2024-03-21T12:18:45.131 INFO:root:teuthology version: 0.0.1.dev262+ge691533 2024-03-21T12:18:45.132 DEBUG:teuthology.run:Teuthology command: teuthology --verbose --name gabrioux-2024-03-21_12:04:17-orch:cephadm-wip-guits-testing-2024-03-21-0818-distro-default-smithi --archive /home/teuthworker/archive/gabrioux-2024-03-21_12:04:17-orch:cephadm-wip-guits-testing-2024-03-21-0818-distro-default-smithi/7614906 --owner scheduled_gabrioux@teuthology --description orch:cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} -- /home/teuthworker/archive/gabrioux-2024-03-21_12:04:17-orch:cephadm-wip-guits-testing-2024-03-21-0818-distro-default-smithi/7614906/orig.config.yaml 2024-03-21T12:18:45.166 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-03-21T12:18:45.247 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/gabrioux-2024-03-21_12:04:17-orch:cephadm-wip-guits-testing-2024-03-21-0818-distro-default-smithi/7614906 branch: wip-guits-testing-2024-03-21-0818 description: orch:cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} email: gabrioux@ibm.com first_in_suite: false job_id: '7614906' kernel: kdb: true sha1: distro last_in_suite: false machine_type: smithi name: gabrioux-2024-03-21_12:04:17-orch:cephadm-wip-guits-testing-2024-03-21-0818-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 9.stream overrides: admin_socket: branch: wip-guits-testing-2024-03-21-0818 ceph: conf: mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: 960fbbe4645c2becc8084669be194c95183c7753 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: 960fbbe4645c2becc8084669be194c95183c7753 selinux: allowlist: - scontext=system_u:system_r:logrotate_t:s0 - scontext=system_u:system_r:getty_t:s0 workunit: branch: wip-guits-testing-2024-03-21-0818 sha1: 960fbbe4645c2becc8084669be194c95183c7753 owner: scheduled_gabrioux@teuthology priority: 70 repo: https://git.ceph.com/ceph-ci.git roles: - - host.a - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - - host.b - osd.4 - osd.5 - osd.6 - osd.7 seed: 187 sha1: 960fbbe4645c2becc8084669be194c95183c7753 sleep_before_teardown: 0 subset: 1/8 suite: orch:cephadm suite_branch: wip-guits-testing-2024-03-21-0818 suite_path: /home/teuthworker/src/git.ceph.com_ceph-c_960fbbe4645c2becc8084669be194c95183c7753/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph-ci.git suite_sha1: 960fbbe4645c2becc8084669be194c95183c7753 targets: smithi022.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNFK97Gjbcjkog67MSuD2palHNl1CaoatkSnBfnz9+hmxYkihXU5tCdYaXONFICPAVaz/2hoRyUkE1ipRratRvs= smithi189.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFBzxp04flUe9qN0scflFfrmoI5QsLDu1LV/+YvcQGrlB/wdkJlgGkGZe2WGMvHTr7uT0lZPSwP1zDMmjuAIkLc= tasks: - cephadm: cephadm_branch: v17.2.0 cephadm_git_url: https://github.com/ceph/ceph image: quay.io/ceph/ceph:v17.2.0 roleless: true - cephadm.shell: host.a: - ceph orch status - ceph orch ps - ceph orch ls - ceph orch host ls - ceph orch device ls - vip.exec: all-hosts: - systemctl stop nfs-server - cephadm.shell: host.a: - ceph fs volume create foofs - cephadm.wait_for_service: service: mds.foofs - cephadm.shell: host.a: - ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2 - ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake - while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done - vip.exec: host.a: - mkdir /mnt/foo - while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done - echo test > /mnt/foo/testfile - sync - parallel: - upgrade-tasks - workload-tasks - vip.exec: host.a: - umount /mnt/foo - cephadm.shell: host.a: - ceph nfs cluster ls | grep foo - ceph nfs export ls foo --detailed - rados -p .nfs --all ls - - cephadm.shell: host.a: - 'set -ex [[ `ceph config get mgr mgr/cephadm/migration_current` -gt 2 ]] ' teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: e691533f9cbb33d85b2187bba20d7102f098636d timestamp: 2024-03-21_12:04:17 tube: smithi upgrade-tasks: sequential: - cephadm.shell: env: - sha1 host.a: - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph config set global log_to_journald false --force - ceph mgr module enable nfs --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - cephadm.shell: env: - sha1 host.a: - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done - ceph orch ps - ceph versions - echo "wait for servicemap items w/ changing names to refresh" - sleep 60 - ceph orch ps - ceph orch upgrade status - ceph health detail - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 - cephadm.wait_for_service: service: nfs.foo user: gabrioux verbose: true worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2226885 workload-tasks: sequential: - exec: host.a: - cd /mnt/foo && dbench 5 -t 600 || true - umount /mnt/foo - while ! mount -t nfs $(hostname):/fake /mnt/foo ; do sleep 5 ; done - cd /mnt/foo && dbench 5 -t 5 2024-03-21T12:18:45.247 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph-c_960fbbe4645c2becc8084669be194c95183c7753/qa; will attempt to use it 2024-03-21T12:18:45.248 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph-c_960fbbe4645c2becc8084669be194c95183c7753/qa/tasks 2024-03-21T12:18:45.248 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-03-21T12:18:45.250 INFO:teuthology.task.internal:Checking packages... 2024-03-21T12:18:45.276 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash '960fbbe4645c2becc8084669be194c95183c7753' 2024-03-21T12:18:45.277 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-03-21T12:18:45.277 INFO:teuthology.packaging:ref: None 2024-03-21T12:18:45.277 INFO:teuthology.packaging:tag: None 2024-03-21T12:18:45.277 INFO:teuthology.packaging:branch: wip-guits-testing-2024-03-21-0818 2024-03-21T12:18:45.277 INFO:teuthology.packaging:sha1: 960fbbe4645c2becc8084669be194c95183c7753 2024-03-21T12:18:45.277 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=wip-guits-testing-2024-03-21-0818 2024-03-21T12:18:45.495 INFO:teuthology.task.internal:Found packages for ceph version 19.0.0-2267.g960fbbe4 2024-03-21T12:18:45.496 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-03-21T12:18:45.503 INFO:teuthology.task.internal:no buildpackages task found 2024-03-21T12:18:45.503 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-03-21T12:18:45.509 INFO:teuthology.task.internal:Saving configuration 2024-03-21T12:18:45.523 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-03-21T12:18:45.529 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-03-21T12:18:45.554 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi022.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/gabrioux-2024-03-21_12:04:17-orch:cephadm-wip-guits-testing-2024-03-21-0818-distro-default-smithi/7614906', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-03-21 12:12:13.938943', 'locked_by': 'scheduled_gabrioux@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNFK97Gjbcjkog67MSuD2palHNl1CaoatkSnBfnz9+hmxYkihXU5tCdYaXONFICPAVaz/2hoRyUkE1ipRratRvs='} 2024-03-21T12:18:45.576 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi189.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/gabrioux-2024-03-21_12:04:17-orch:cephadm-wip-guits-testing-2024-03-21-0818-distro-default-smithi/7614906', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-03-21 12:12:13.940154', 'locked_by': 'scheduled_gabrioux@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFBzxp04flUe9qN0scflFfrmoI5QsLDu1LV/+YvcQGrlB/wdkJlgGkGZe2WGMvHTr7uT0lZPSwP1zDMmjuAIkLc='} 2024-03-21T12:18:45.576 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-03-21T12:18:45.582 INFO:teuthology.task.internal:roles: ubuntu@smithi022.front.sepia.ceph.com - ['host.a', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0'] 2024-03-21T12:18:45.582 INFO:teuthology.task.internal:roles: ubuntu@smithi189.front.sepia.ceph.com - ['host.b', 'osd.4', 'osd.5', 'osd.6', 'osd.7'] 2024-03-21T12:18:45.582 INFO:teuthology.run_tasks:Running task console_log... 2024-03-21T12:18:45.652 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7fe294c14f70>, signals=[15]) 2024-03-21T12:18:45.653 INFO:teuthology.run_tasks:Running task internal.connect... 2024-03-21T12:18:45.661 INFO:teuthology.task.internal:Opening connections... 2024-03-21T12:18:45.661 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi022.front.sepia.ceph.com 2024-03-21T12:18:45.663 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi022.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-21T12:18:45.769 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi189.front.sepia.ceph.com 2024-03-21T12:18:45.770 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi189.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-21T12:18:45.855 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-03-21T12:18:45.861 DEBUG:teuthology.orchestra.run.smithi022:> uname -m 2024-03-21T12:18:45.879 INFO:teuthology.orchestra.run.smithi022.stdout:x86_64 2024-03-21T12:18:45.879 DEBUG:teuthology.orchestra.run.smithi022:> cat /etc/os-release 2024-03-21T12:18:45.936 INFO:teuthology.orchestra.run.smithi022.stdout:NAME="CentOS Stream" 2024-03-21T12:18:45.936 INFO:teuthology.orchestra.run.smithi022.stdout:VERSION="9" 2024-03-21T12:18:45.936 INFO:teuthology.orchestra.run.smithi022.stdout:ID="centos" 2024-03-21T12:18:45.936 INFO:teuthology.orchestra.run.smithi022.stdout:ID_LIKE="rhel fedora" 2024-03-21T12:18:45.936 INFO:teuthology.orchestra.run.smithi022.stdout:VERSION_ID="9" 2024-03-21T12:18:45.936 INFO:teuthology.orchestra.run.smithi022.stdout:PLATFORM_ID="platform:el9" 2024-03-21T12:18:45.937 INFO:teuthology.orchestra.run.smithi022.stdout:PRETTY_NAME="CentOS Stream 9" 2024-03-21T12:18:45.937 INFO:teuthology.orchestra.run.smithi022.stdout:ANSI_COLOR="0;31" 2024-03-21T12:18:45.937 INFO:teuthology.orchestra.run.smithi022.stdout:LOGO="fedora-logo-icon" 2024-03-21T12:18:45.937 INFO:teuthology.orchestra.run.smithi022.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-03-21T12:18:45.937 INFO:teuthology.orchestra.run.smithi022.stdout:HOME_URL="https://centos.org/" 2024-03-21T12:18:45.937 INFO:teuthology.orchestra.run.smithi022.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-03-21T12:18:45.937 INFO:teuthology.orchestra.run.smithi022.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-03-21T12:18:45.937 INFO:teuthology.orchestra.run.smithi022.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-03-21T12:18:45.937 INFO:teuthology.lock.ops:Updating smithi022.front.sepia.ceph.com on lock server 2024-03-21T12:18:45.955 DEBUG:teuthology.orchestra.run.smithi189:> uname -m 2024-03-21T12:18:45.972 INFO:teuthology.orchestra.run.smithi189.stdout:x86_64 2024-03-21T12:18:45.972 DEBUG:teuthology.orchestra.run.smithi189:> cat /etc/os-release 2024-03-21T12:18:46.029 INFO:teuthology.orchestra.run.smithi189.stdout:NAME="CentOS Stream" 2024-03-21T12:18:46.029 INFO:teuthology.orchestra.run.smithi189.stdout:VERSION="9" 2024-03-21T12:18:46.029 INFO:teuthology.orchestra.run.smithi189.stdout:ID="centos" 2024-03-21T12:18:46.029 INFO:teuthology.orchestra.run.smithi189.stdout:ID_LIKE="rhel fedora" 2024-03-21T12:18:46.029 INFO:teuthology.orchestra.run.smithi189.stdout:VERSION_ID="9" 2024-03-21T12:18:46.029 INFO:teuthology.orchestra.run.smithi189.stdout:PLATFORM_ID="platform:el9" 2024-03-21T12:18:46.029 INFO:teuthology.orchestra.run.smithi189.stdout:PRETTY_NAME="CentOS Stream 9" 2024-03-21T12:18:46.029 INFO:teuthology.orchestra.run.smithi189.stdout:ANSI_COLOR="0;31" 2024-03-21T12:18:46.029 INFO:teuthology.orchestra.run.smithi189.stdout:LOGO="fedora-logo-icon" 2024-03-21T12:18:46.029 INFO:teuthology.orchestra.run.smithi189.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-03-21T12:18:46.029 INFO:teuthology.orchestra.run.smithi189.stdout:HOME_URL="https://centos.org/" 2024-03-21T12:18:46.029 INFO:teuthology.orchestra.run.smithi189.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-03-21T12:18:46.029 INFO:teuthology.orchestra.run.smithi189.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-03-21T12:18:46.030 INFO:teuthology.orchestra.run.smithi189.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-03-21T12:18:46.030 INFO:teuthology.lock.ops:Updating smithi189.front.sepia.ceph.com on lock server 2024-03-21T12:18:46.218 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-03-21T12:18:46.226 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-03-21T12:18:46.232 INFO:teuthology.task.internal:Checking for old test directory... 2024-03-21T12:18:46.232 DEBUG:teuthology.orchestra.run.smithi022:> test '!' -e /home/ubuntu/cephtest 2024-03-21T12:18:46.235 DEBUG:teuthology.orchestra.run.smithi189:> test '!' -e /home/ubuntu/cephtest 2024-03-21T12:18:46.251 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-03-21T12:18:46.256 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-03-21T12:18:46.257 DEBUG:teuthology.orchestra.run.smithi022:> test -z $(ls -A /var/lib/ceph) 2024-03-21T12:18:46.292 DEBUG:teuthology.orchestra.run.smithi189:> test -z $(ls -A /var/lib/ceph) 2024-03-21T12:18:46.321 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-03-21T12:18:46.375 INFO:teuthology.run_tasks:Running task kernel... 2024-03-21T12:18:46.389 INFO:teuthology.task.kernel:normalize config orig: {'kdb': True, 'sha1': 'distro'} 2024-03-21T12:18:46.389 INFO:teuthology.task.kernel:config {'host.a': {'kdb': True, 'sha1': 'distro'}, 'host.b': {'kdb': True, 'sha1': 'distro'}}, timeout 300 2024-03-21T12:18:46.390 DEBUG:teuthology.orchestra.run.smithi022:> test -f /run/.containerenv -o -f /.dockerenv 2024-03-21T12:18:46.390 DEBUG:teuthology.orchestra.run.smithi189:> test -f /run/.containerenv -o -f /.dockerenv 2024-03-21T12:18:46.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-21T12:18:46.406 DEBUG:teuthology.orchestra.run.smithi189:> uname -r 2024-03-21T12:18:46.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-21T12:18:46.407 DEBUG:teuthology.orchestra.run.smithi022:> uname -r 2024-03-21T12:18:46.462 INFO:teuthology.orchestra.run.smithi189.stdout:5.14.0-361.el9.x86_64 2024-03-21T12:18:46.463 INFO:teuthology.task.kernel:Running kernel on smithi189: 5.14.0-361.el9.x86_64 2024-03-21T12:18:46.463 DEBUG:teuthology.orchestra.run.smithi189:> sudo yum install -y kernel 2024-03-21T12:18:46.466 INFO:teuthology.orchestra.run.smithi022.stdout:5.14.0-361.el9.x86_64 2024-03-21T12:18:46.466 INFO:teuthology.task.kernel:Running kernel on smithi022: 5.14.0-361.el9.x86_64 2024-03-21T12:18:46.466 DEBUG:teuthology.orchestra.run.smithi022:> sudo yum install -y kernel 2024-03-21T12:18:48.832 INFO:teuthology.orchestra.run.smithi189.stdout:CentOS-9 - AppStream 9.1 MB/s | 6.7 MB 00:00 2024-03-21T12:18:50.624 INFO:teuthology.orchestra.run.smithi189.stdout:CentOS-9 - BaseOS 3.3 MB/s | 2.0 MB 00:00 2024-03-21T12:18:50.999 INFO:teuthology.orchestra.run.smithi022.stdout:CentOS-9 - AppStream 2.3 MB/s | 6.7 MB 00:02 2024-03-21T12:18:52.056 INFO:teuthology.orchestra.run.smithi189.stdout:CentOS Stream 9 - BaseOS 8.0 MB/s | 8.0 MB 00:01 2024-03-21T12:18:52.432 INFO:teuthology.orchestra.run.smithi022.stdout:CentOS-9 - BaseOS 7.0 MB/s | 2.0 MB 00:00 2024-03-21T12:18:53.757 INFO:teuthology.orchestra.run.smithi189.stdout:CentOS Stream 9 - AppStream 31 MB/s | 19 MB 00:00 2024-03-21T12:18:53.764 INFO:teuthology.orchestra.run.smithi022.stdout:CentOS Stream 9 - BaseOS 8.2 MB/s | 8.0 MB 00:00 2024-03-21T12:18:55.657 INFO:teuthology.orchestra.run.smithi022.stdout:CentOS Stream 9 - AppStream 24 MB/s | 19 MB 00:00 2024-03-21T12:18:57.816 INFO:teuthology.orchestra.run.smithi189.stdout:CentOS Stream 9 - CRB 12 MB/s | 6.1 MB 00:00 2024-03-21T12:19:00.017 INFO:teuthology.orchestra.run.smithi189.stdout:CentOS Stream 9 - Extras packages 15 kB/s | 16 kB 00:01 2024-03-21T12:19:01.660 INFO:teuthology.orchestra.run.smithi189.stdout:Extra Packages for Enterprise Linux 14 MB/s | 21 MB 00:01 2024-03-21T12:19:06.576 INFO:teuthology.orchestra.run.smithi189.stdout:lab-extras 15 kB/s | 1.7 kB 00:00 2024-03-21T12:19:08.173 INFO:teuthology.orchestra.run.smithi022.stdout:CentOS Stream 9 - CRB 708 kB/s | 6.1 MB 00:08 2024-03-21T12:19:08.646 INFO:teuthology.orchestra.run.smithi189.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-03-21T12:19:08.646 INFO:teuthology.orchestra.run.smithi189.stdout:Package kernel-5.14.0-361.el9.x86_64 is already installed. 2024-03-21T12:19:08.729 INFO:teuthology.orchestra.run.smithi189.stdout:Dependencies resolved. 2024-03-21T12:19:08.744 INFO:teuthology.orchestra.run.smithi189.stdout:================================================================================ 2024-03-21T12:19:08.744 INFO:teuthology.orchestra.run.smithi189.stdout: Package Architecture Version Repository Size 2024-03-21T12:19:08.745 INFO:teuthology.orchestra.run.smithi189.stdout:================================================================================ 2024-03-21T12:19:08.745 INFO:teuthology.orchestra.run.smithi189.stdout:Installing: 2024-03-21T12:19:08.745 INFO:teuthology.orchestra.run.smithi189.stdout: kernel x86_64 5.14.0-430.el9 baseos 44 k 2024-03-21T12:19:08.745 INFO:teuthology.orchestra.run.smithi189.stdout:Installing dependencies: 2024-03-21T12:19:08.745 INFO:teuthology.orchestra.run.smithi189.stdout: kernel-core x86_64 5.14.0-430.el9 baseos 15 M 2024-03-21T12:19:08.745 INFO:teuthology.orchestra.run.smithi189.stdout: kernel-modules x86_64 5.14.0-430.el9 baseos 33 M 2024-03-21T12:19:08.745 INFO:teuthology.orchestra.run.smithi189.stdout: kernel-modules-core x86_64 5.14.0-430.el9 baseos 27 M 2024-03-21T12:19:08.745 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-03-21T12:19:08.745 INFO:teuthology.orchestra.run.smithi189.stdout:Transaction Summary 2024-03-21T12:19:08.745 INFO:teuthology.orchestra.run.smithi189.stdout:================================================================================ 2024-03-21T12:19:08.746 INFO:teuthology.orchestra.run.smithi189.stdout:Install 4 Packages 2024-03-21T12:19:08.746 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-03-21T12:19:08.746 INFO:teuthology.orchestra.run.smithi189.stdout:Total download size: 76 M 2024-03-21T12:19:08.746 INFO:teuthology.orchestra.run.smithi189.stdout:Installed size: 123 M 2024-03-21T12:19:08.746 INFO:teuthology.orchestra.run.smithi189.stdout:Downloading Packages: 2024-03-21T12:19:09.120 INFO:teuthology.orchestra.run.smithi189.stdout:(1/4): kernel-5.14.0-430.el9.x86_64.rpm 191 kB/s | 44 kB 00:00 2024-03-21T12:19:09.790 INFO:teuthology.orchestra.run.smithi022.stdout:CentOS Stream 9 - Extras packages 37 kB/s | 16 kB 00:00 2024-03-21T12:19:11.025 INFO:teuthology.orchestra.run.smithi022.stdout:Extra Packages for Enterprise Linux 19 MB/s | 21 MB 00:01 2024-03-21T12:19:11.578 INFO:teuthology.orchestra.run.smithi189.stdout:(2/4): kernel-modules-core-5.14.0-430.el9.x86_6 11 MB/s | 27 MB 00:02 2024-03-21T12:19:11.728 INFO:teuthology.orchestra.run.smithi189.stdout:(3/4): kernel-core-5.14.0-430.el9.x86_64.rpm 5.2 MB/s | 15 MB 00:02 2024-03-21T12:19:12.861 INFO:teuthology.orchestra.run.smithi189.stdout:(4/4): kernel-modules-5.14.0-430.el9.x86_64.rpm 8.4 MB/s | 33 MB 00:03 2024-03-21T12:19:12.864 INFO:teuthology.orchestra.run.smithi189.stdout:-------------------------------------------------------------------------------- 2024-03-21T12:19:12.864 INFO:teuthology.orchestra.run.smithi189.stdout:Total 18 MB/s | 76 MB 00:04 2024-03-21T12:19:13.534 INFO:teuthology.orchestra.run.smithi189.stdout:Running transaction check 2024-03-21T12:19:13.875 INFO:teuthology.orchestra.run.smithi189.stdout:Transaction check succeeded. 2024-03-21T12:19:13.875 INFO:teuthology.orchestra.run.smithi189.stdout:Running transaction test 2024-03-21T12:19:14.868 INFO:teuthology.orchestra.run.smithi189.stdout:Transaction test succeeded. 2024-03-21T12:19:14.869 INFO:teuthology.orchestra.run.smithi189.stdout:Running transaction 2024-03-21T12:19:16.150 INFO:teuthology.orchestra.run.smithi022.stdout:lab-extras 31 kB/s | 1.7 kB 00:00 2024-03-21T12:19:17.362 INFO:teuthology.orchestra.run.smithi189.stdout: Preparing : 1/1 2024-03-21T12:19:18.049 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : kernel-modules-core-5.14.0-430.el9.x86_64 1/4 2024-03-21T12:19:18.145 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : kernel-core-5.14.0-430.el9.x86_64 2/4 2024-03-21T12:19:18.340 INFO:teuthology.orchestra.run.smithi022.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-03-21T12:19:18.340 INFO:teuthology.orchestra.run.smithi022.stdout:Package kernel-5.14.0-361.el9.x86_64 is already installed. 2024-03-21T12:19:18.442 INFO:teuthology.orchestra.run.smithi022.stdout:Dependencies resolved. 2024-03-21T12:19:18.453 INFO:teuthology.orchestra.run.smithi022.stdout:================================================================================ 2024-03-21T12:19:18.454 INFO:teuthology.orchestra.run.smithi022.stdout: Package Architecture Version Repository Size 2024-03-21T12:19:18.454 INFO:teuthology.orchestra.run.smithi022.stdout:================================================================================ 2024-03-21T12:19:18.454 INFO:teuthology.orchestra.run.smithi022.stdout:Installing: 2024-03-21T12:19:18.454 INFO:teuthology.orchestra.run.smithi022.stdout: kernel x86_64 5.14.0-430.el9 baseos 44 k 2024-03-21T12:19:18.454 INFO:teuthology.orchestra.run.smithi022.stdout:Installing dependencies: 2024-03-21T12:19:18.454 INFO:teuthology.orchestra.run.smithi022.stdout: kernel-core x86_64 5.14.0-430.el9 baseos 15 M 2024-03-21T12:19:18.454 INFO:teuthology.orchestra.run.smithi022.stdout: kernel-modules x86_64 5.14.0-430.el9 baseos 33 M 2024-03-21T12:19:18.454 INFO:teuthology.orchestra.run.smithi022.stdout: kernel-modules-core x86_64 5.14.0-430.el9 baseos 27 M 2024-03-21T12:19:18.454 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-03-21T12:19:18.455 INFO:teuthology.orchestra.run.smithi022.stdout:Transaction Summary 2024-03-21T12:19:18.455 INFO:teuthology.orchestra.run.smithi022.stdout:================================================================================ 2024-03-21T12:19:18.455 INFO:teuthology.orchestra.run.smithi022.stdout:Install 4 Packages 2024-03-21T12:19:18.455 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-03-21T12:19:18.455 INFO:teuthology.orchestra.run.smithi022.stdout:Total download size: 76 M 2024-03-21T12:19:18.455 INFO:teuthology.orchestra.run.smithi022.stdout:Installed size: 123 M 2024-03-21T12:19:18.455 INFO:teuthology.orchestra.run.smithi022.stdout:Downloading Packages: 2024-03-21T12:19:18.868 INFO:teuthology.orchestra.run.smithi022.stdout:(1/4): kernel-5.14.0-430.el9.x86_64.rpm 127 kB/s | 44 kB 00:00 2024-03-21T12:19:20.919 INFO:teuthology.orchestra.run.smithi189.stdout: Running scriptlet: kernel-core-5.14.0-430.el9.x86_64 2/4 2024-03-21T12:19:21.047 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : kernel-modules-5.14.0-430.el9.x86_64 3/4 2024-03-21T12:19:25.786 INFO:teuthology.orchestra.run.smithi189.stdout: Running scriptlet: kernel-modules-5.14.0-430.el9.x86_64 3/4 2024-03-21T12:19:25.899 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : kernel-5.14.0-430.el9.x86_64 4/4 2024-03-21T12:19:29.409 INFO:teuthology.orchestra.run.smithi022.stdout:(2/4): kernel-core-5.14.0-430.el9.x86_64.rpm 1.4 MB/s | 15 MB 00:10 2024-03-21T12:19:30.653 INFO:teuthology.orchestra.run.smithi189.stdout: Running scriptlet: kernel-modules-core-5.14.0-430.el9.x86_64 4/4 2024-03-21T12:19:30.825 INFO:teuthology.orchestra.run.smithi022.stdout:(3/4): kernel-modules-core-5.14.0-430.el9.x86_6 2.3 MB/s | 27 MB 00:11 2024-03-21T12:19:35.175 INFO:teuthology.orchestra.run.smithi022.stdout:(4/4): kernel-modules-5.14.0-430.el9.x86_64.rpm 2.0 MB/s | 33 MB 00:16 2024-03-21T12:19:35.175 INFO:teuthology.orchestra.run.smithi022.stdout:-------------------------------------------------------------------------------- 2024-03-21T12:19:35.176 INFO:teuthology.orchestra.run.smithi022.stdout:Total 4.5 MB/s | 76 MB 00:16 2024-03-21T12:19:35.870 INFO:teuthology.orchestra.run.smithi022.stdout:Running transaction check 2024-03-21T12:19:36.299 INFO:teuthology.orchestra.run.smithi022.stdout:Transaction check succeeded. 2024-03-21T12:19:36.299 INFO:teuthology.orchestra.run.smithi022.stdout:Running transaction test 2024-03-21T12:19:37.328 INFO:teuthology.orchestra.run.smithi022.stdout:Transaction test succeeded. 2024-03-21T12:19:37.328 INFO:teuthology.orchestra.run.smithi022.stdout:Running transaction 2024-03-21T12:19:39.957 INFO:teuthology.orchestra.run.smithi022.stdout: Preparing : 1/1 2024-03-21T12:19:40.579 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : kernel-modules-core-5.14.0-430.el9.x86_64 1/4 2024-03-21T12:19:40.675 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : kernel-core-5.14.0-430.el9.x86_64 2/4 2024-03-21T12:19:43.562 INFO:teuthology.orchestra.run.smithi022.stdout: Running scriptlet: kernel-core-5.14.0-430.el9.x86_64 2/4 2024-03-21T12:19:43.695 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : kernel-modules-5.14.0-430.el9.x86_64 3/4 2024-03-21T12:19:48.761 INFO:teuthology.orchestra.run.smithi022.stdout: Running scriptlet: kernel-modules-5.14.0-430.el9.x86_64 3/4 2024-03-21T12:19:48.805 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : kernel-5.14.0-430.el9.x86_64 4/4 2024-03-21T12:19:53.908 INFO:teuthology.orchestra.run.smithi022.stdout: Running scriptlet: kernel-modules-core-5.14.0-430.el9.x86_64 4/4 2024-03-21T12:19:59.774 INFO:teuthology.orchestra.run.smithi189.stdout: Running scriptlet: kernel-core-5.14.0-430.el9.x86_64 4/4 2024-03-21T12:19:59.774 INFO:teuthology.orchestra.run.smithi189.stdout:dracut: Disabling early microcode, because kernel does not support it. CONFIG_MICROCODE_[AMD|INTEL]!=y 2024-03-21T12:19:59.774 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-03-21T12:19:59.819 INFO:teuthology.orchestra.run.smithi189.stdout: Running scriptlet: kernel-modules-5.14.0-430.el9.x86_64 4/4 2024-03-21T12:20:00.142 INFO:teuthology.orchestra.run.smithi189.stdout: Running scriptlet: kernel-5.14.0-430.el9.x86_64 4/4 2024-03-21T12:20:00.142 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : kernel-5.14.0-430.el9.x86_64 1/4 2024-03-21T12:20:00.142 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : kernel-core-5.14.0-430.el9.x86_64 2/4 2024-03-21T12:20:00.142 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : kernel-modules-5.14.0-430.el9.x86_64 3/4 2024-03-21T12:20:00.540 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : kernel-modules-core-5.14.0-430.el9.x86_64 4/4 2024-03-21T12:20:00.540 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-03-21T12:20:00.540 INFO:teuthology.orchestra.run.smithi189.stdout:Installed: 2024-03-21T12:20:00.540 INFO:teuthology.orchestra.run.smithi189.stdout: kernel-5.14.0-430.el9.x86_64 2024-03-21T12:20:00.540 INFO:teuthology.orchestra.run.smithi189.stdout: kernel-core-5.14.0-430.el9.x86_64 2024-03-21T12:20:00.540 INFO:teuthology.orchestra.run.smithi189.stdout: kernel-modules-5.14.0-430.el9.x86_64 2024-03-21T12:20:00.540 INFO:teuthology.orchestra.run.smithi189.stdout: kernel-modules-core-5.14.0-430.el9.x86_64 2024-03-21T12:20:00.540 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-03-21T12:20:00.541 INFO:teuthology.orchestra.run.smithi189.stdout:Complete! 2024-03-21T12:20:00.855 DEBUG:teuthology.orchestra.run.smithi189:> rpm -q kernel | sort -rV | head -n 1 2024-03-21T12:20:00.967 INFO:teuthology.orchestra.run.smithi189.stdout:kernel-5.14.0-430.el9.x86_64 2024-03-21T12:20:00.968 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-430.el9.x86_64 2024-03-21T12:20:00.968 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-361.el9.x86_64 Expected: 5.14.0-430.el9.x86_64 2024-03-21T12:20:00.968 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-03-21T12:20:00.968 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-03-21T12:20:00.968 INFO:teuthology.task.kernel:Installing distro kernel on host.b... 2024-03-21T12:20:00.968 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi189.front.sepia.ceph.com, path=None, version=distro) 2024-03-21T12:20:00.969 DEBUG:teuthology.orchestra.run.smithi189:> rpm -q kernel | sort -rV | head -n 1 2024-03-21T12:20:01.048 INFO:teuthology.orchestra.run.smithi189.stdout:kernel-5.14.0-430.el9.x86_64 2024-03-21T12:20:01.048 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-430.el9.x86_64 2024-03-21T12:20:01.049 DEBUG:teuthology.orchestra.run.smithi189:> sudo rpm -qi grub2-tools 2024-03-21T12:20:01.083 INFO:teuthology.orchestra.run.smithi189.stdout:Name : grub2-tools 2024-03-21T12:20:01.083 INFO:teuthology.orchestra.run.smithi189.stdout:Epoch : 1 2024-03-21T12:20:01.084 INFO:teuthology.orchestra.run.smithi189.stdout:Version : 2.06 2024-03-21T12:20:01.084 INFO:teuthology.orchestra.run.smithi189.stdout:Release : 68.el9 2024-03-21T12:20:01.084 INFO:teuthology.orchestra.run.smithi189.stdout:Architecture: x86_64 2024-03-21T12:20:01.084 INFO:teuthology.orchestra.run.smithi189.stdout:Install Date: Fri 01 Sep 2023 09:14:30 PM UTC 2024-03-21T12:20:01.084 INFO:teuthology.orchestra.run.smithi189.stdout:Group : Unspecified 2024-03-21T12:20:01.084 INFO:teuthology.orchestra.run.smithi189.stdout:Size : 8264254 2024-03-21T12:20:01.084 INFO:teuthology.orchestra.run.smithi189.stdout:License : GPLv3+ 2024-03-21T12:20:01.084 INFO:teuthology.orchestra.run.smithi189.stdout:Signature : RSA/SHA256, Mon 07 Aug 2023 01:57:16 PM UTC, Key ID 05b555b38483c65d 2024-03-21T12:20:01.084 INFO:teuthology.orchestra.run.smithi189.stdout:Source RPM : grub2-2.06-68.el9.src.rpm 2024-03-21T12:20:01.084 INFO:teuthology.orchestra.run.smithi189.stdout:Build Date : Wed 02 Aug 2023 03:15:48 PM UTC 2024-03-21T12:20:01.084 INFO:teuthology.orchestra.run.smithi189.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-03-21T12:20:01.084 INFO:teuthology.orchestra.run.smithi189.stdout:Packager : builder@centos.org 2024-03-21T12:20:01.084 INFO:teuthology.orchestra.run.smithi189.stdout:Vendor : CentOS 2024-03-21T12:20:01.085 INFO:teuthology.orchestra.run.smithi189.stdout:URL : http://www.gnu.org/software/grub/ 2024-03-21T12:20:01.085 INFO:teuthology.orchestra.run.smithi189.stdout:Summary : Support tools for GRUB. 2024-03-21T12:20:01.085 INFO:teuthology.orchestra.run.smithi189.stdout:Description : 2024-03-21T12:20:01.085 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-03-21T12:20:01.085 INFO:teuthology.orchestra.run.smithi189.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-03-21T12:20:01.085 INFO:teuthology.orchestra.run.smithi189.stdout:customizable bootloader with modular architecture. It supports a rich 2024-03-21T12:20:01.085 INFO:teuthology.orchestra.run.smithi189.stdout:variety of kernel formats, file systems, computer architectures and 2024-03-21T12:20:01.085 INFO:teuthology.orchestra.run.smithi189.stdout:hardware devices. 2024-03-21T12:20:01.085 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-03-21T12:20:01.085 INFO:teuthology.orchestra.run.smithi189.stdout:This subpackage provides tools for support of all platforms. 2024-03-21T12:20:01.086 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-03-21T12:20:01.086 INFO:teuthology.task.kernel:Updating grub on smithi189 to boot 5.14.0-430.el9.x86_64 2024-03-21T12:20:01.086 DEBUG:teuthology.orchestra.run.smithi189:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-03-21T12:20:01.683 INFO:teuthology.orchestra.run.smithi189.stderr:Generating grub configuration file ... 2024-03-21T12:20:03.124 INFO:teuthology.orchestra.run.smithi189.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-03-21T12:20:03.154 INFO:teuthology.orchestra.run.smithi189.stderr:done 2024-03-21T12:20:03.157 DEBUG:teuthology.orchestra.run.smithi189:> mktemp 2024-03-21T12:20:03.171 INFO:teuthology.orchestra.run.smithi189.stdout:/tmp/tmp.gUgdIlJmLY 2024-03-21T12:20:03.172 DEBUG:teuthology.orchestra.run.smithi189:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.gUgdIlJmLY 2024-03-21T12:20:03.240 DEBUG:teuthology.orchestra.run.smithi189:> sudo chmod 0666 /tmp/tmp.gUgdIlJmLY 2024-03-21T12:20:03.381 DEBUG:teuthology.orchestra.remote:smithi189:/tmp/tmp.gUgdIlJmLY is 6KB 2024-03-21T12:20:03.401 DEBUG:teuthology.orchestra.run.smithi189:> rm -fr /tmp/tmp.gUgdIlJmLY 2024-03-21T12:20:03.415 DEBUG:teuthology.orchestra.run.smithi189:> sudo /bin/ls /boot/loader/entries || true 2024-03-21T12:20:03.480 INFO:teuthology.orchestra.run.smithi189.stdout:15949e3c88704300882482eddfd7baae-0-rescue.conf 2024-03-21T12:20:03.481 INFO:teuthology.orchestra.run.smithi189.stdout:15949e3c88704300882482eddfd7baae-5.14.0-239.el9.x86_64.conf 2024-03-21T12:20:03.481 INFO:teuthology.orchestra.run.smithi189.stdout:15949e3c88704300882482eddfd7baae-5.14.0-361.el9.x86_64.conf 2024-03-21T12:20:03.481 INFO:teuthology.orchestra.run.smithi189.stdout:15949e3c88704300882482eddfd7baae-5.14.0-430.el9.x86_64.conf 2024-03-21T12:20:03.482 DEBUG:teuthology.orchestra.run.smithi189:> sudo grub2-set-default 15949e3c88704300882482eddfd7baae-5.14.0-430.el9.x86_64 2024-03-21T12:20:03.638 DEBUG:teuthology.orchestra.run.smithi189:> sudo shutdown -r now 2024-03-21T12:20:24.656 INFO:teuthology.orchestra.run.smithi022.stdout: Running scriptlet: kernel-core-5.14.0-430.el9.x86_64 4/4 2024-03-21T12:20:24.657 INFO:teuthology.orchestra.run.smithi022.stdout:dracut: Disabling early microcode, because kernel does not support it. CONFIG_MICROCODE_[AMD|INTEL]!=y 2024-03-21T12:20:24.657 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-03-21T12:20:24.702 INFO:teuthology.orchestra.run.smithi022.stdout: Running scriptlet: kernel-modules-5.14.0-430.el9.x86_64 4/4 2024-03-21T12:20:25.075 INFO:teuthology.orchestra.run.smithi022.stdout: Running scriptlet: kernel-5.14.0-430.el9.x86_64 4/4 2024-03-21T12:20:25.075 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : kernel-5.14.0-430.el9.x86_64 1/4 2024-03-21T12:20:25.075 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : kernel-core-5.14.0-430.el9.x86_64 2/4 2024-03-21T12:20:25.075 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : kernel-modules-5.14.0-430.el9.x86_64 3/4 2024-03-21T12:20:25.491 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : kernel-modules-core-5.14.0-430.el9.x86_64 4/4 2024-03-21T12:20:25.491 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-03-21T12:20:25.491 INFO:teuthology.orchestra.run.smithi022.stdout:Installed: 2024-03-21T12:20:25.491 INFO:teuthology.orchestra.run.smithi022.stdout: kernel-5.14.0-430.el9.x86_64 2024-03-21T12:20:25.491 INFO:teuthology.orchestra.run.smithi022.stdout: kernel-core-5.14.0-430.el9.x86_64 2024-03-21T12:20:25.491 INFO:teuthology.orchestra.run.smithi022.stdout: kernel-modules-5.14.0-430.el9.x86_64 2024-03-21T12:20:25.492 INFO:teuthology.orchestra.run.smithi022.stdout: kernel-modules-core-5.14.0-430.el9.x86_64 2024-03-21T12:20:25.492 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-03-21T12:20:25.492 INFO:teuthology.orchestra.run.smithi022.stdout:Complete! 2024-03-21T12:20:25.813 DEBUG:teuthology.orchestra.run.smithi022:> rpm -q kernel | sort -rV | head -n 1 2024-03-21T12:20:25.915 INFO:teuthology.orchestra.run.smithi022.stdout:kernel-5.14.0-430.el9.x86_64 2024-03-21T12:20:25.918 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-430.el9.x86_64 2024-03-21T12:20:25.918 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-361.el9.x86_64 Expected: 5.14.0-430.el9.x86_64 2024-03-21T12:20:25.918 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-03-21T12:20:25.918 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-03-21T12:20:25.918 INFO:teuthology.task.kernel:Installing distro kernel on host.a... 2024-03-21T12:20:25.919 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi022.front.sepia.ceph.com, path=None, version=distro) 2024-03-21T12:20:25.919 DEBUG:teuthology.orchestra.run.smithi022:> rpm -q kernel | sort -rV | head -n 1 2024-03-21T12:20:26.047 INFO:teuthology.orchestra.run.smithi022.stdout:kernel-5.14.0-430.el9.x86_64 2024-03-21T12:20:26.047 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-430.el9.x86_64 2024-03-21T12:20:26.047 DEBUG:teuthology.orchestra.run.smithi022:> sudo rpm -qi grub2-tools 2024-03-21T12:20:26.087 INFO:teuthology.orchestra.run.smithi022.stdout:Name : grub2-tools 2024-03-21T12:20:26.087 INFO:teuthology.orchestra.run.smithi022.stdout:Epoch : 1 2024-03-21T12:20:26.087 INFO:teuthology.orchestra.run.smithi022.stdout:Version : 2.06 2024-03-21T12:20:26.087 INFO:teuthology.orchestra.run.smithi022.stdout:Release : 68.el9 2024-03-21T12:20:26.087 INFO:teuthology.orchestra.run.smithi022.stdout:Architecture: x86_64 2024-03-21T12:20:26.087 INFO:teuthology.orchestra.run.smithi022.stdout:Install Date: Fri 01 Sep 2023 09:14:30 PM UTC 2024-03-21T12:20:26.087 INFO:teuthology.orchestra.run.smithi022.stdout:Group : Unspecified 2024-03-21T12:20:26.087 INFO:teuthology.orchestra.run.smithi022.stdout:Size : 8264254 2024-03-21T12:20:26.088 INFO:teuthology.orchestra.run.smithi022.stdout:License : GPLv3+ 2024-03-21T12:20:26.088 INFO:teuthology.orchestra.run.smithi022.stdout:Signature : RSA/SHA256, Mon 07 Aug 2023 01:57:16 PM UTC, Key ID 05b555b38483c65d 2024-03-21T12:20:26.088 INFO:teuthology.orchestra.run.smithi022.stdout:Source RPM : grub2-2.06-68.el9.src.rpm 2024-03-21T12:20:26.088 INFO:teuthology.orchestra.run.smithi022.stdout:Build Date : Wed 02 Aug 2023 03:15:48 PM UTC 2024-03-21T12:20:26.088 INFO:teuthology.orchestra.run.smithi022.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-03-21T12:20:26.088 INFO:teuthology.orchestra.run.smithi022.stdout:Packager : builder@centos.org 2024-03-21T12:20:26.088 INFO:teuthology.orchestra.run.smithi022.stdout:Vendor : CentOS 2024-03-21T12:20:26.088 INFO:teuthology.orchestra.run.smithi022.stdout:URL : http://www.gnu.org/software/grub/ 2024-03-21T12:20:26.088 INFO:teuthology.orchestra.run.smithi022.stdout:Summary : Support tools for GRUB. 2024-03-21T12:20:26.088 INFO:teuthology.orchestra.run.smithi022.stdout:Description : 2024-03-21T12:20:26.088 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-03-21T12:20:26.088 INFO:teuthology.orchestra.run.smithi022.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-03-21T12:20:26.089 INFO:teuthology.orchestra.run.smithi022.stdout:customizable bootloader with modular architecture. It supports a rich 2024-03-21T12:20:26.089 INFO:teuthology.orchestra.run.smithi022.stdout:variety of kernel formats, file systems, computer architectures and 2024-03-21T12:20:26.089 INFO:teuthology.orchestra.run.smithi022.stdout:hardware devices. 2024-03-21T12:20:26.089 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-03-21T12:20:26.089 INFO:teuthology.orchestra.run.smithi022.stdout:This subpackage provides tools for support of all platforms. 2024-03-21T12:20:26.090 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-03-21T12:20:26.090 INFO:teuthology.task.kernel:Updating grub on smithi022 to boot 5.14.0-430.el9.x86_64 2024-03-21T12:20:26.090 DEBUG:teuthology.orchestra.run.smithi022:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-03-21T12:20:26.699 INFO:teuthology.orchestra.run.smithi022.stderr:Generating grub configuration file ... 2024-03-21T12:20:28.021 INFO:teuthology.orchestra.run.smithi022.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-03-21T12:20:28.052 INFO:teuthology.orchestra.run.smithi022.stderr:done 2024-03-21T12:20:28.054 DEBUG:teuthology.orchestra.run.smithi022:> mktemp 2024-03-21T12:20:28.070 INFO:teuthology.orchestra.run.smithi022.stdout:/tmp/tmp.EO1OY6QZcy 2024-03-21T12:20:28.070 DEBUG:teuthology.orchestra.run.smithi022:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.EO1OY6QZcy 2024-03-21T12:20:28.142 DEBUG:teuthology.orchestra.run.smithi022:> sudo chmod 0666 /tmp/tmp.EO1OY6QZcy 2024-03-21T12:20:28.282 DEBUG:teuthology.orchestra.remote:smithi022:/tmp/tmp.EO1OY6QZcy is 6KB 2024-03-21T12:20:28.331 DEBUG:teuthology.orchestra.run.smithi022:> rm -fr /tmp/tmp.EO1OY6QZcy 2024-03-21T12:20:28.345 DEBUG:teuthology.orchestra.run.smithi022:> sudo /bin/ls /boot/loader/entries || true 2024-03-21T12:20:28.420 INFO:teuthology.orchestra.run.smithi022.stdout:15949e3c88704300882482eddfd7baae-0-rescue.conf 2024-03-21T12:20:28.420 INFO:teuthology.orchestra.run.smithi022.stdout:15949e3c88704300882482eddfd7baae-5.14.0-239.el9.x86_64.conf 2024-03-21T12:20:28.420 INFO:teuthology.orchestra.run.smithi022.stdout:15949e3c88704300882482eddfd7baae-5.14.0-361.el9.x86_64.conf 2024-03-21T12:20:28.420 INFO:teuthology.orchestra.run.smithi022.stdout:15949e3c88704300882482eddfd7baae-5.14.0-430.el9.x86_64.conf 2024-03-21T12:20:28.421 DEBUG:teuthology.orchestra.run.smithi022:> sudo grub2-set-default 15949e3c88704300882482eddfd7baae-5.14.0-430.el9.x86_64 2024-03-21T12:20:28.561 DEBUG:teuthology.orchestra.run.smithi022:> sudo shutdown -r now 2024-03-21T12:20:33.642 INFO:teuthology.task.kernel:Checking client host.b for new kernel version... 2024-03-21T12:20:33.643 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi189.front.sepia.ceph.com' 2024-03-21T12:20:33.643 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi189.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-21T12:20:58.585 INFO:teuthology.task.kernel:Checking client host.a for new kernel version... 2024-03-21T12:20:58.585 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi022.front.sepia.ceph.com' 2024-03-21T12:20:58.586 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi022.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-21T12:21:08.258 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.189 2024-03-21T12:21:17.267 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi189.front.sepia.ceph.com' 2024-03-21T12:21:17.268 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi189.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-21T12:21:35.651 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.189 2024-03-21T12:21:47.662 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi189.front.sepia.ceph.com' 2024-03-21T12:21:47.663 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi189.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-21T12:21:50.718 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.189 2024-03-21T12:21:58.595 DEBUG:teuthology.orchestra.remote:timed out 2024-03-21T12:22:05.726 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi189.front.sepia.ceph.com' 2024-03-21T12:22:05.727 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi189.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-21T12:22:06.354 DEBUG:teuthology.orchestra.run.smithi189:> true 2024-03-21T12:22:06.755 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi189.front.sepia.ceph.com' 2024-03-21T12:22:06.755 INFO:teuthology.task.kernel:Checking kernel version of host.b, want "5.14.0-430.el9.x86_64"... 2024-03-21T12:22:06.756 DEBUG:teuthology.orchestra.run.smithi189:> uname -r 2024-03-21T12:22:06.772 INFO:teuthology.orchestra.run.smithi189.stdout:5.14.0-430.el9.x86_64 2024-03-21T12:22:06.772 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-430.el9.x86_64 vs 5.14.0-430.el9.x86_64 2024-03-21T12:22:06.772 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-03-21T12:22:06.772 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-03-21T12:22:07.596 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi022.front.sepia.ceph.com' 2024-03-21T12:22:07.597 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi022.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-21T12:22:07.773 INFO:teuthology.task.kernel:Enabling kdb on host.b... 2024-03-21T12:22:07.774 DEBUG:teuthology.orchestra.run.smithi189:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-03-21T12:22:07.839 INFO:teuthology.orchestra.run.smithi189.stdout:ttyS1 2024-03-21T12:22:07.887 DEBUG:teuthology.parallel:result is None 2024-03-21T12:22:08.009 DEBUG:teuthology.orchestra.run.smithi022:> true 2024-03-21T12:22:08.400 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi022.front.sepia.ceph.com' 2024-03-21T12:22:08.400 INFO:teuthology.task.kernel:Checking kernel version of host.a, want "5.14.0-430.el9.x86_64"... 2024-03-21T12:22:08.400 DEBUG:teuthology.orchestra.run.smithi022:> uname -r 2024-03-21T12:22:08.414 INFO:teuthology.orchestra.run.smithi022.stdout:5.14.0-430.el9.x86_64 2024-03-21T12:22:08.414 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-430.el9.x86_64 vs 5.14.0-430.el9.x86_64 2024-03-21T12:22:08.414 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-03-21T12:22:08.414 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-03-21T12:22:09.415 INFO:teuthology.task.kernel:Enabling kdb on host.a... 2024-03-21T12:22:09.415 DEBUG:teuthology.orchestra.run.smithi022:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-03-21T12:22:09.492 INFO:teuthology.orchestra.run.smithi022.stdout:ttyS1 2024-03-21T12:22:09.526 DEBUG:teuthology.parallel:result is None 2024-03-21T12:22:09.529 INFO:teuthology.run_tasks:Running task internal.base... 2024-03-21T12:22:09.536 INFO:teuthology.task.internal:Creating test directory... 2024-03-21T12:22:09.537 DEBUG:teuthology.orchestra.run.smithi022:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-03-21T12:22:09.539 DEBUG:teuthology.orchestra.run.smithi189:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-03-21T12:22:09.556 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-03-21T12:22:09.563 INFO:teuthology.run_tasks:Running task internal.archive... 2024-03-21T12:22:09.582 INFO:teuthology.task.internal:Creating archive directory... 2024-03-21T12:22:09.582 DEBUG:teuthology.orchestra.run.smithi022:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-03-21T12:22:09.591 DEBUG:teuthology.orchestra.run.smithi189:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-03-21T12:22:09.638 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-03-21T12:22:09.645 INFO:teuthology.task.internal:Enabling coredump saving... 2024-03-21T12:22:09.645 DEBUG:teuthology.orchestra.run.smithi022:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-03-21T12:22:09.659 DEBUG:teuthology.orchestra.run.smithi189:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-03-21T12:22:09.701 INFO:teuthology.orchestra.run.smithi022.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-03-21T12:22:09.709 INFO:teuthology.orchestra.run.smithi189.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-03-21T12:22:09.715 INFO:teuthology.orchestra.run.smithi022.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-03-21T12:22:09.723 INFO:teuthology.orchestra.run.smithi189.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-03-21T12:22:09.724 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-03-21T12:22:09.731 INFO:teuthology.task.internal:Configuring sudo... 2024-03-21T12:22:09.732 DEBUG:teuthology.orchestra.run.smithi022:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-03-21T12:22:09.734 DEBUG:teuthology.orchestra.run.smithi189:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-03-21T12:22:09.794 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-03-21T12:22:09.803 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-03-21T12:22:09.803 DEBUG:teuthology.orchestra.run.smithi022:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-03-21T12:22:09.805 DEBUG:teuthology.orchestra.run.smithi189:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-03-21T12:22:09.850 DEBUG:teuthology.orchestra.run.smithi022:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-03-21T12:22:09.901 DEBUG:teuthology.orchestra.run.smithi022:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-03-21T12:22:09.955 DEBUG:teuthology.orchestra.run.smithi022:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-03-21T12:22:10.004 DEBUG:teuthology.orchestra.run.smithi022:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-03-21T12:22:10.067 DEBUG:teuthology.orchestra.run.smithi022:> set -ex 2024-03-21T12:22:10.068 DEBUG:teuthology.orchestra.run.smithi022:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-03-21T12:22:10.107 DEBUG:teuthology.orchestra.run.smithi189:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-03-21T12:22:10.151 DEBUG:teuthology.orchestra.run.smithi189:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-03-21T12:22:10.199 DEBUG:teuthology.orchestra.run.smithi189:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-03-21T12:22:10.283 DEBUG:teuthology.orchestra.run.smithi189:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-03-21T12:22:10.326 DEBUG:teuthology.orchestra.run.smithi189:> set -ex 2024-03-21T12:22:10.326 DEBUG:teuthology.orchestra.run.smithi189:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-03-21T12:22:10.394 DEBUG:teuthology.orchestra.run.smithi022:> sudo service rsyslog restart 2024-03-21T12:22:10.398 DEBUG:teuthology.orchestra.run.smithi189:> sudo service rsyslog restart 2024-03-21T12:22:10.468 INFO:teuthology.orchestra.run.smithi022.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-03-21T12:22:10.483 INFO:teuthology.orchestra.run.smithi189.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-03-21T12:22:10.993 INFO:teuthology.run_tasks:Running task internal.timer... 2024-03-21T12:22:11.001 INFO:teuthology.task.internal:Starting timer... 2024-03-21T12:22:11.001 INFO:teuthology.run_tasks:Running task pcp... 2024-03-21T12:22:11.011 INFO:teuthology.run_tasks:Running task selinux... 2024-03-21T12:22:11.021 DEBUG:teuthology.task:Applying overrides for task selinux: {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0', 'scontext=system_u:system_r:getty_t:s0']} 2024-03-21T12:22:11.022 DEBUG:teuthology.orchestra.run.smithi022:> sudo service auditd rotate 2024-03-21T12:22:11.077 INFO:teuthology.orchestra.run.smithi022.stdout:Rotating logs: 2024-03-21T12:22:11.079 DEBUG:teuthology.orchestra.run.smithi189:> sudo service auditd rotate 2024-03-21T12:22:11.136 INFO:teuthology.orchestra.run.smithi189.stdout:Rotating logs: 2024-03-21T12:22:11.138 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-03-21T12:22:11.138 DEBUG:teuthology.orchestra.run.smithi022:> /usr/sbin/getenforce 2024-03-21T12:22:11.166 INFO:teuthology.orchestra.run.smithi022.stdout:Permissive 2024-03-21T12:22:11.167 DEBUG:teuthology.orchestra.run.smithi189:> /usr/sbin/getenforce 2024-03-21T12:22:11.234 INFO:teuthology.orchestra.run.smithi189.stdout:Permissive 2024-03-21T12:22:11.234 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi022.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi189.front.sepia.ceph.com': 'permissive'} 2024-03-21T12:22:11.235 DEBUG:teuthology.orchestra.run.smithi022:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-03-21T12:22:11.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-21T12:22:11.263 DEBUG:teuthology.orchestra.run.smithi189:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-03-21T12:22:11.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-21T12:22:11.291 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-03-21T12:22:11.292 DEBUG:teuthology.orchestra.run.smithi022:> sudo /usr/sbin/setenforce permissive 2024-03-21T12:22:11.414 DEBUG:teuthology.orchestra.run.smithi189:> sudo /usr/sbin/setenforce permissive 2024-03-21T12:22:11.444 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-03-21T12:22:11.455 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-03-21T12:22:11.472 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-03-21T12:22:11.473 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi022.front.sepia.ceph.com,smithi189.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-03-21T12:26:31.030 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi022.front.sepia.ceph.com'), Remote(name='ubuntu@smithi189.front.sepia.ceph.com')] 2024-03-21T12:26:31.031 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi022.front.sepia.ceph.com' 2024-03-21T12:26:31.033 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi022.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-21T12:26:31.127 DEBUG:teuthology.orchestra.run.smithi022:> true 2024-03-21T12:26:31.205 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi022.front.sepia.ceph.com' 2024-03-21T12:26:31.206 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi189.front.sepia.ceph.com' 2024-03-21T12:26:31.206 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi189.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-21T12:26:31.293 DEBUG:teuthology.orchestra.run.smithi189:> true 2024-03-21T12:26:31.369 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi189.front.sepia.ceph.com' 2024-03-21T12:26:31.370 INFO:teuthology.run_tasks:Running task clock... 2024-03-21T12:26:31.380 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-03-21T12:26:31.380 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-03-21T12:26:31.381 DEBUG:teuthology.orchestra.run.smithi022:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-03-21T12:26:31.383 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-03-21T12:26:31.383 DEBUG:teuthology.orchestra.run.smithi189:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-03-21T12:26:31.414 INFO:teuthology.orchestra.run.smithi022.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-03-21T12:26:31.431 INFO:teuthology.orchestra.run.smithi022.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-03-21T12:26:31.441 INFO:teuthology.orchestra.run.smithi189.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-03-21T12:26:31.458 INFO:teuthology.orchestra.run.smithi189.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-03-21T12:26:31.461 INFO:teuthology.orchestra.run.smithi022.stderr:sudo: ntpd: command not found 2024-03-21T12:26:31.475 INFO:teuthology.orchestra.run.smithi022.stdout:506 Cannot talk to daemon 2024-03-21T12:26:31.487 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: ntpd: command not found 2024-03-21T12:26:31.491 INFO:teuthology.orchestra.run.smithi022.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-03-21T12:26:31.501 INFO:teuthology.orchestra.run.smithi189.stdout:506 Cannot talk to daemon 2024-03-21T12:26:31.507 INFO:teuthology.orchestra.run.smithi022.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-03-21T12:26:31.518 INFO:teuthology.orchestra.run.smithi189.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-03-21T12:26:31.534 INFO:teuthology.orchestra.run.smithi189.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-03-21T12:26:31.560 INFO:teuthology.orchestra.run.smithi022.stderr:bash: line 1: ntpq: command not found 2024-03-21T12:26:31.564 INFO:teuthology.orchestra.run.smithi022.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-03-21T12:26:31.564 INFO:teuthology.orchestra.run.smithi022.stdout:=============================================================================== 2024-03-21T12:26:31.564 INFO:teuthology.orchestra.run.smithi022.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-03-21T12:26:31.564 INFO:teuthology.orchestra.run.smithi022.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-03-21T12:26:31.564 INFO:teuthology.orchestra.run.smithi022.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-03-21T12:26:31.564 INFO:teuthology.orchestra.run.smithi022.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-03-21T12:26:31.588 INFO:teuthology.orchestra.run.smithi189.stderr:bash: line 1: ntpq: command not found 2024-03-21T12:26:31.595 INFO:teuthology.orchestra.run.smithi189.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-03-21T12:26:31.595 INFO:teuthology.orchestra.run.smithi189.stdout:=============================================================================== 2024-03-21T12:26:31.595 INFO:teuthology.orchestra.run.smithi189.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-03-21T12:26:31.595 INFO:teuthology.orchestra.run.smithi189.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-03-21T12:26:31.595 INFO:teuthology.orchestra.run.smithi189.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-03-21T12:26:31.595 INFO:teuthology.orchestra.run.smithi189.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-03-21T12:26:31.595 INFO:teuthology.run_tasks:Running task cephadm... 2024-03-21T12:26:31.702 INFO:tasks.cephadm:Config: {'cephadm_branch': 'v17.2.0', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'image': 'quay.io/ceph/ceph:v17.2.0', 'roleless': True, 'conf': {'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': '960fbbe4645c2becc8084669be194c95183c7753'} 2024-03-21T12:26:31.702 INFO:tasks.cephadm:Cluster image is quay.io/ceph/ceph:v17.2.0 2024-03-21T12:26:31.702 INFO:tasks.cephadm:Cluster fsid is 3fc7f0f2-e77e-11ee-95cd-87774f69a715 2024-03-21T12:26:31.702 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-03-21T12:26:31.703 INFO:tasks.cephadm:No mon roles; fabricating mons 2024-03-21T12:26:31.703 INFO:tasks.cephadm:Monitor IPs: {'mon.smithi022': '172.21.15.22', 'mon.smithi189': '172.21.15.189'} 2024-03-21T12:26:31.703 INFO:tasks.cephadm:Normalizing hostnames... 2024-03-21T12:26:31.703 DEBUG:teuthology.orchestra.run.smithi022:> sudo hostname $(hostname -s) 2024-03-21T12:26:31.730 DEBUG:teuthology.orchestra.run.smithi189:> sudo hostname $(hostname -s) 2024-03-21T12:26:31.759 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref v17.2.0)... 2024-03-21T12:26:31.759 DEBUG:teuthology.orchestra.run.smithi022:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-03-21T12:26:32.145 INFO:teuthology.orchestra.run.smithi022.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Mar 21 12:26 /home/ubuntu/cephtest/cephadm 2024-03-21T12:26:32.145 DEBUG:teuthology.orchestra.run.smithi189:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-03-21T12:26:32.491 INFO:teuthology.orchestra.run.smithi189.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Mar 21 12:26 /home/ubuntu/cephtest/cephadm 2024-03-21T12:26:32.496 DEBUG:teuthology.orchestra.run.smithi022:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-03-21T12:26:32.518 DEBUG:teuthology.orchestra.run.smithi189:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-03-21T12:26:32.544 INFO:tasks.cephadm:Pulling image quay.io/ceph/ceph:v17.2.0 on all hosts... 2024-03-21T12:26:32.545 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-03-21T12:26:32.566 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-03-21T12:26:33.420 INFO:teuthology.orchestra.run.smithi189.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-03-21T12:26:33.458 INFO:teuthology.orchestra.run.smithi022.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-03-21T12:26:59.512 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-03-21T12:26:59.513 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-03-21T12:26:59.513 INFO:teuthology.orchestra.run.smithi022.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-03-21T12:26:59.513 INFO:teuthology.orchestra.run.smithi022.stdout: "repo_digests": [ 2024-03-21T12:26:59.513 INFO:teuthology.orchestra.run.smithi022.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-03-21T12:26:59.513 INFO:teuthology.orchestra.run.smithi022.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-03-21T12:26:59.513 INFO:teuthology.orchestra.run.smithi022.stdout: ] 2024-03-21T12:26:59.513 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-03-21T12:26:59.721 INFO:teuthology.orchestra.run.smithi189.stdout:{ 2024-03-21T12:26:59.721 INFO:teuthology.orchestra.run.smithi189.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-03-21T12:26:59.721 INFO:teuthology.orchestra.run.smithi189.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-03-21T12:26:59.721 INFO:teuthology.orchestra.run.smithi189.stdout: "repo_digests": [ 2024-03-21T12:26:59.721 INFO:teuthology.orchestra.run.smithi189.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-03-21T12:26:59.721 INFO:teuthology.orchestra.run.smithi189.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-03-21T12:26:59.721 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-03-21T12:26:59.721 INFO:teuthology.orchestra.run.smithi189.stdout:} 2024-03-21T12:26:59.740 DEBUG:teuthology.orchestra.run.smithi022:> sudo mkdir -p /etc/ceph 2024-03-21T12:26:59.771 DEBUG:teuthology.orchestra.run.smithi189:> sudo mkdir -p /etc/ceph 2024-03-21T12:26:59.801 DEBUG:teuthology.orchestra.run.smithi022:> sudo chmod 777 /etc/ceph 2024-03-21T12:26:59.838 DEBUG:teuthology.orchestra.run.smithi189:> sudo chmod 777 /etc/ceph 2024-03-21T12:26:59.868 INFO:tasks.cephadm:Writing seed config... 2024-03-21T12:26:59.870 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-03-21T12:26:59.870 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-03-21T12:26:59.870 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-03-21T12:26:59.870 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-03-21T12:26:59.870 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-03-21T12:26:59.870 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-03-21T12:26:59.870 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-03-21T12:26:59.870 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2024-03-21T12:26:59.871 DEBUG:teuthology.orchestra.run.smithi022:> set -ex 2024-03-21T12:26:59.871 DEBUG:teuthology.orchestra.run.smithi022:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-03-21T12:26:59.894 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 3fc7f0f2-e77e-11ee-95cd-87774f69a715 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-03-21T12:26:59.894 DEBUG:teuthology.orchestra.run.smithi022:mon.smithi022> sudo journalctl -f -n 0 -u ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715@mon.smithi022.service 2024-03-21T12:26:59.937 INFO:tasks.cephadm:Bootstrapping... 2024-03-21T12:26:59.937 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 -v bootstrap --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-ip 172.21.15.22 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-03-21T12:27:00.137 INFO:teuthology.orchestra.run.smithi022.stderr:-------------------------------------------------------------------------------- 2024-03-21T12:27:00.137 INFO:teuthology.orchestra.run.smithi022.stderr:cephadm ['--image', 'quay.io/ceph/ceph:v17.2.0', '-v', 'bootstrap', '--fsid', '3fc7f0f2-e77e-11ee-95cd-87774f69a715', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-ip', '172.21.15.22', '--skip-admin-label'] 2024-03-21T12:27:00.160 INFO:teuthology.orchestra.run.smithi022.stderr:/bin/podman: 4.9.3-dev 2024-03-21T12:27:00.162 INFO:teuthology.orchestra.run.smithi022.stderr:Verifying podman|docker is present... 2024-03-21T12:27:00.183 INFO:teuthology.orchestra.run.smithi022.stderr:/bin/podman: 4.9.3-dev 2024-03-21T12:27:00.185 INFO:teuthology.orchestra.run.smithi022.stderr:Verifying lvm2 is present... 2024-03-21T12:27:00.185 INFO:teuthology.orchestra.run.smithi022.stderr:Verifying time synchronization is in place... 2024-03-21T12:27:00.192 INFO:teuthology.orchestra.run.smithi022.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-03-21T12:27:00.198 INFO:teuthology.orchestra.run.smithi022.stderr:systemctl: inactive 2024-03-21T12:27:00.205 INFO:teuthology.orchestra.run.smithi022.stderr:systemctl: enabled 2024-03-21T12:27:00.210 INFO:teuthology.orchestra.run.smithi022.stderr:systemctl: active 2024-03-21T12:27:00.211 INFO:teuthology.orchestra.run.smithi022.stderr:Unit chronyd.service is enabled and running 2024-03-21T12:27:00.211 INFO:teuthology.orchestra.run.smithi022.stderr:Repeating the final host check... 2024-03-21T12:27:00.231 INFO:teuthology.orchestra.run.smithi022.stderr:/bin/podman: 4.9.3-dev 2024-03-21T12:27:00.233 INFO:teuthology.orchestra.run.smithi022.stderr:podman (/bin/podman) version 4.9.3 is present 2024-03-21T12:27:00.233 INFO:teuthology.orchestra.run.smithi022.stderr:systemctl is present 2024-03-21T12:27:00.234 INFO:teuthology.orchestra.run.smithi022.stderr:lvcreate is present 2024-03-21T12:27:00.239 INFO:teuthology.orchestra.run.smithi022.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-03-21T12:27:00.245 INFO:teuthology.orchestra.run.smithi022.stderr:systemctl: inactive 2024-03-21T12:27:00.251 INFO:teuthology.orchestra.run.smithi022.stderr:systemctl: enabled 2024-03-21T12:27:00.257 INFO:teuthology.orchestra.run.smithi022.stderr:systemctl: active 2024-03-21T12:27:00.258 INFO:teuthology.orchestra.run.smithi022.stderr:Unit chronyd.service is enabled and running 2024-03-21T12:27:00.258 INFO:teuthology.orchestra.run.smithi022.stderr:Host looks OK 2024-03-21T12:27:00.258 INFO:teuthology.orchestra.run.smithi022.stderr:Cluster fsid: 3fc7f0f2-e77e-11ee-95cd-87774f69a715 2024-03-21T12:27:00.258 INFO:teuthology.orchestra.run.smithi022.stderr:Acquiring lock 139695233891728 on /run/cephadm/3fc7f0f2-e77e-11ee-95cd-87774f69a715.lock 2024-03-21T12:27:00.259 INFO:teuthology.orchestra.run.smithi022.stderr:Lock 139695233891728 acquired on /run/cephadm/3fc7f0f2-e77e-11ee-95cd-87774f69a715.lock 2024-03-21T12:27:00.259 INFO:teuthology.orchestra.run.smithi022.stderr:Verifying IP 172.21.15.22 port 3300 ... 2024-03-21T12:27:00.259 INFO:teuthology.orchestra.run.smithi022.stderr:Verifying IP 172.21.15.22 port 6789 ... 2024-03-21T12:27:00.259 INFO:teuthology.orchestra.run.smithi022.stderr:Base mon IP is 172.21.15.22, final addrv is [v2:172.21.15.22:3300,v1:172.21.15.22:6789] 2024-03-21T12:27:00.261 INFO:teuthology.orchestra.run.smithi022.stderr:/sbin/ip: default via 172.21.15.254 dev ens1f0 proto dhcp src 172.21.15.22 metric 100 2024-03-21T12:27:00.261 INFO:teuthology.orchestra.run.smithi022.stderr:/sbin/ip: 172.21.0.0/20 dev ens1f0 proto kernel scope link src 172.21.15.22 metric 100 2024-03-21T12:27:00.268 INFO:teuthology.orchestra.run.smithi022.stderr:/sbin/ip: ::1 dev lo proto kernel metric 256 pref medium 2024-03-21T12:27:00.268 INFO:teuthology.orchestra.run.smithi022.stderr:/sbin/ip: fe80::/64 dev ens1f0 proto kernel metric 1024 pref medium 2024-03-21T12:27:00.268 INFO:teuthology.orchestra.run.smithi022.stderr:/sbin/ip: default via fe80::327c:5e00:6487:71e0 dev ens1f0 proto ra metric 100 pref medium 2024-03-21T12:27:00.268 INFO:teuthology.orchestra.run.smithi022.stderr:/sbin/ip: 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-03-21T12:27:00.268 INFO:teuthology.orchestra.run.smithi022.stderr:/sbin/ip: inet6 ::1/128 scope host 2024-03-21T12:27:00.268 INFO:teuthology.orchestra.run.smithi022.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-03-21T12:27:00.268 INFO:teuthology.orchestra.run.smithi022.stderr:/sbin/ip: 4: ens1f0: mtu 1500 state UP qlen 1000 2024-03-21T12:27:00.268 INFO:teuthology.orchestra.run.smithi022.stderr:/sbin/ip: inet6 fe80::ec4:7aff:fe59:5de4/64 scope link noprefixroute 2024-03-21T12:27:00.268 INFO:teuthology.orchestra.run.smithi022.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-03-21T12:27:00.268 INFO:teuthology.orchestra.run.smithi022.stderr:Mon IP `172.21.15.22` is in CIDR network `172.21.0.0/20` 2024-03-21T12:27:00.268 INFO:teuthology.orchestra.run.smithi022.stderr:- internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-03-21T12:27:00.269 INFO:teuthology.orchestra.run.smithi022.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-03-21T12:27:00.371 INFO:teuthology.orchestra.run.smithi022.stderr:/bin/podman: Trying to pull quay.io/ceph/ceph:v17.2.0... 2024-03-21T12:27:00.817 INFO:teuthology.orchestra.run.smithi022.stderr:/bin/podman: Getting image source signatures 2024-03-21T12:27:00.817 INFO:teuthology.orchestra.run.smithi022.stderr:/bin/podman: Copying blob sha256:33ca8fff7868c4dc0c11e09bca97c720eb9cfbab7221216754367dd8de70388a 2024-03-21T12:27:00.817 INFO:teuthology.orchestra.run.smithi022.stderr:/bin/podman: Copying blob sha256:a70843738bb77e1ab9c1f85969ebdfa55f178e746be081d1cb4f94011f69eb7c 2024-03-21T12:27:00.817 INFO:teuthology.orchestra.run.smithi022.stderr:/bin/podman: Copying blob sha256:c32ab78b488d0c72f64eded765c0cf6b5bf2c75dab66cb62a9d367fa6ec42513 2024-03-21T12:27:00.817 INFO:teuthology.orchestra.run.smithi022.stderr:/bin/podman: Copying blob sha256:89b4a75bc2d8500f15463747507c9623df43886c134463e7f0527e70900e7a7b 2024-03-21T12:27:00.817 INFO:teuthology.orchestra.run.smithi022.stderr:/bin/podman: Copying blob sha256:599d07cb321ff0a3c82224e1138fc685793fa69b93ed5780415751a5f7e4b8c2 2024-03-21T12:27:00.817 INFO:teuthology.orchestra.run.smithi022.stderr:/bin/podman: Copying config sha256:e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-03-21T12:27:00.932 INFO:teuthology.orchestra.run.smithi022.stderr:/bin/podman: Writing manifest to image destination 2024-03-21T12:27:01.035 INFO:teuthology.orchestra.run.smithi022.stderr:/bin/podman: e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-03-21T12:27:01.418 INFO:teuthology.orchestra.run.smithi022.stderr:ceph: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-03-21T12:27:01.892 INFO:teuthology.orchestra.run.smithi022.stderr:Ceph version: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-03-21T12:27:01.892 INFO:teuthology.orchestra.run.smithi022.stderr:Extracting ceph user uid/gid from container image... 2024-03-21T12:27:02.291 INFO:teuthology.orchestra.run.smithi022.stderr:stat: 167 167 2024-03-21T12:27:02.906 INFO:teuthology.orchestra.run.smithi022.stderr:Creating initial keys... 2024-03-21T12:27:03.322 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph-authtool: AQCXJ/xlj4eqDRAAsy3SZH3A0OnT5JUIPzR8Kw== 2024-03-21T12:27:04.219 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph-authtool: AQCYJ/xln72GBxAAeRC6P4q9gesv9FNN01YmHw== 2024-03-21T12:27:05.190 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph-authtool: AQCZJ/xl6MJuBxAAoUjN/JAANeimYwmORT2X5w== 2024-03-21T12:27:05.681 INFO:teuthology.orchestra.run.smithi022.stderr:Creating initial monmap... 2024-03-21T12:27:06.121 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: monmap file /tmp/monmap 2024-03-21T12:27:06.121 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/monmaptool: setting min_mon_release = octopus 2024-03-21T12:27:06.122 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: set fsid to 3fc7f0f2-e77e-11ee-95cd-87774f69a715 2024-03-21T12:27:06.122 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-03-21T12:27:06.686 INFO:teuthology.orchestra.run.smithi022.stderr:monmaptool for smithi022 [v2:172.21.15.22:3300,v1:172.21.15.22:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-03-21T12:27:06.686 INFO:teuthology.orchestra.run.smithi022.stderr:setting min_mon_release = octopus 2024-03-21T12:27:06.686 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/monmaptool: set fsid to 3fc7f0f2-e77e-11ee-95cd-87774f69a715 2024-03-21T12:27:06.686 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-03-21T12:27:06.686 INFO:teuthology.orchestra.run.smithi022.stderr: 2024-03-21T12:27:06.686 INFO:teuthology.orchestra.run.smithi022.stderr:Creating mon... 2024-03-21T12:27:07.599 INFO:teuthology.orchestra.run.smithi022.stderr:create mon.smithi022 on 2024-03-21T12:27:07.800 INFO:teuthology.orchestra.run.smithi022.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-03-21T12:27:07.977 INFO:teuthology.orchestra.run.smithi022.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715.target → /etc/systemd/system/ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715.target. 2024-03-21T12:27:07.978 INFO:teuthology.orchestra.run.smithi022.stderr:systemctl: Created symlink /etc/systemd/system/ceph.target.wants/ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715.target → /etc/systemd/system/ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715.target. 2024-03-21T12:27:08.264 INFO:teuthology.orchestra.run.smithi022.stderr:systemctl: Failed to reset failed state of unit ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715@mon.smithi022.service: Unit ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715@mon.smithi022.service not loaded. 2024-03-21T12:27:08.271 INFO:teuthology.orchestra.run.smithi022.stderr:systemctl: Created symlink /etc/systemd/system/ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715.target.wants/ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715@mon.smithi022.service → /etc/systemd/system/ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715@.service. 2024-03-21T12:27:08.935 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:08 smithi022 ceph-mon[24420]: mon.smithi022 is new leader, mons smithi022 in quorum (ranks 0) 2024-03-21T12:27:08.935 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:08 smithi022 bash[24401]: fc88aed1393b13d97c4c2cbbeb82fb31f30f75b8c547f6d6e6fc8f7afb69f2d7 2024-03-21T12:27:08.946 INFO:teuthology.orchestra.run.smithi022.stderr:systemctl: disabled 2024-03-21T12:27:08.952 INFO:teuthology.orchestra.run.smithi022.stderr:systemctl: inactive 2024-03-21T12:27:08.953 INFO:teuthology.orchestra.run.smithi022.stderr:firewalld.service is not enabled 2024-03-21T12:27:08.953 INFO:teuthology.orchestra.run.smithi022.stderr:Not possible to enable service . firewalld.service is not available 2024-03-21T12:27:08.953 INFO:teuthology.orchestra.run.smithi022.stderr:Waiting for mon to start... 2024-03-21T12:27:08.953 INFO:teuthology.orchestra.run.smithi022.stderr:Waiting for mon... 2024-03-21T12:27:09.191 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:08 smithi022 systemd[1]: Started Ceph mon.smithi022 for 3fc7f0f2-e77e-11ee-95cd-87774f69a715. 2024-03-21T12:27:09.464 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: cluster: 2024-03-21T12:27:09.464 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: id: 3fc7f0f2-e77e-11ee-95cd-87774f69a715 2024-03-21T12:27:09.464 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: health: HEALTH_OK 2024-03-21T12:27:09.464 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: 2024-03-21T12:27:09.464 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: services: 2024-03-21T12:27:09.464 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: mon: 1 daemons, quorum smithi022 (age 0.567551s) 2024-03-21T12:27:09.464 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: mgr: no daemons active 2024-03-21T12:27:09.464 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: osd: 0 osds: 0 up, 0 in 2024-03-21T12:27:09.464 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: 2024-03-21T12:27:09.465 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: data: 2024-03-21T12:27:09.465 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: pools: 0 pools, 0 pgs 2024-03-21T12:27:09.465 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: objects: 0 objects, 0 B 2024-03-21T12:27:09.465 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: usage: 0 B used, 0 B / 0 B avail 2024-03-21T12:27:09.465 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: pgs: 2024-03-21T12:27:09.465 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: 2024-03-21T12:27:10.073 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:09 smithi022 ceph-mon[24420]: mon.smithi022 is new leader, mons smithi022 in quorum (ranks 0) 2024-03-21T12:27:10.074 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:09 smithi022 ceph-mon[24420]: monmap e1: 1 mons at {smithi022=[v2:172.21.15.22:3300/0,v1:172.21.15.22:6789/0]} 2024-03-21T12:27:10.074 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:09 smithi022 ceph-mon[24420]: fsmap 2024-03-21T12:27:10.074 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:09 smithi022 ceph-mon[24420]: osdmap e1: 0 total, 0 up, 0 in 2024-03-21T12:27:10.074 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:09 smithi022 ceph-mon[24420]: mgrmap e1: no daemons active 2024-03-21T12:27:10.074 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:09 smithi022 ceph-mon[24420]: from='client.? 172.21.15.22:0/809164479' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-03-21T12:27:10.076 INFO:teuthology.orchestra.run.smithi022.stderr:mon is available 2024-03-21T12:27:10.076 INFO:teuthology.orchestra.run.smithi022.stderr:Assimilating anything we can from ceph.conf... 2024-03-21T12:27:10.576 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: 2024-03-21T12:27:10.576 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: [global] 2024-03-21T12:27:10.576 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: fsid = 3fc7f0f2-e77e-11ee-95cd-87774f69a715 2024-03-21T12:27:10.576 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: mon_host = [v2:172.21.15.22:3300,v1:172.21.15.22:6789] 2024-03-21T12:27:10.576 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-03-21T12:27:10.576 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-03-21T12:27:10.576 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-03-21T12:27:10.576 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-03-21T12:27:10.576 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: 2024-03-21T12:27:10.576 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: [mgr] 2024-03-21T12:27:10.576 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-03-21T12:27:10.577 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: 2024-03-21T12:27:10.577 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: [osd] 2024-03-21T12:27:10.577 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-03-21T12:27:10.577 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-03-21T12:27:11.180 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:10 smithi022 ceph-mon[24420]: from='client.? 172.21.15.22:0/2678803504' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-03-21T12:27:11.180 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:10 smithi022 ceph-mon[24420]: from='client.? 172.21.15.22:0/2678803504' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-03-21T12:27:11.182 INFO:teuthology.orchestra.run.smithi022.stderr:Generating new minimal ceph.conf... 2024-03-21T12:27:12.186 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:11 smithi022 ceph-mon[24420]: from='client.? 172.21.15.22:0/3638211097' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:27:12.189 INFO:teuthology.orchestra.run.smithi022.stderr:Restarting the monitor... 2024-03-21T12:27:12.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:12 smithi022 systemd[1]: Stopping Ceph mon.smithi022 for 3fc7f0f2-e77e-11ee-95cd-87774f69a715... 2024-03-21T12:27:12.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:12 smithi022 bash[24639]: Error: no container with name or ID "ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-mon.smithi022" found: no such container 2024-03-21T12:27:13.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:13 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-mon-smithi022[24416]: 2024-03-21T12:27:13.071+0000 7f71d3a5b700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi022 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-03-21T12:27:13.345 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:13 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-mon-smithi022[24416]: 2024-03-21T12:27:13.071+0000 7f71d3a5b700 -1 mon.smithi022@0(leader) e1 *** Got Signal Terminated *** 2024-03-21T12:27:13.345 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:13 smithi022 podman[24652]: 2024-03-21 12:27:13.143207797 +0000 UTC m=+0.898438664 container died fc88aed1393b13d97c4c2cbbeb82fb31f30f75b8c547f6d6e6fc8f7afb69f2d7 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-mon-smithi022, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, vcs-type=git, version=8, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., ceph=True, maintainer=Guillaume Abrioux , distribution-scope=public, release=754, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD) 2024-03-21T12:27:17.345 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:16 smithi022 podman[24652]: 2024-03-21 12:27:16.9282754 +0000 UTC m=+4.683506256 container cleanup fc88aed1393b13d97c4c2cbbeb82fb31f30f75b8c547f6d6e6fc8f7afb69f2d7 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-mon-smithi022, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, RELEASE=HEAD, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, distribution-scope=public, release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, ceph=True, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-03-21T12:27:17.345 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:16 smithi022 bash[24652]: ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-mon-smithi022 2024-03-21T12:27:17.345 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:16 smithi022 bash[24678]: Error: no container with name or ID "ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-mon.smithi022" found: no such container 2024-03-21T12:27:19.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:18 smithi022 podman[24666]: 2024-03-21 12:27:18.740246569 +0000 UTC m=+5.595828734 container remove fc88aed1393b13d97c4c2cbbeb82fb31f30f75b8c547f6d6e6fc8f7afb69f2d7 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-mon-smithi022, CEPH_POINT_RELEASE=-17.2.0, release=754, build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, GIT_CLEAN=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, ceph=True, version=8, name=centos-stream) 2024-03-21T12:27:19.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:18 smithi022 systemd[1]: ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715@mon.smithi022.service: Deactivated successfully. 2024-03-21T12:27:19.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:18 smithi022 systemd[1]: Stopped Ceph mon.smithi022 for 3fc7f0f2-e77e-11ee-95cd-87774f69a715. 2024-03-21T12:27:19.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:18 smithi022 systemd[1]: Starting Ceph mon.smithi022 for 3fc7f0f2-e77e-11ee-95cd-87774f69a715... 2024-03-21T12:27:19.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:18 smithi022 podman[24747]: 2024-03-21 12:27:18.884098269 +0000 UTC m=+0.018593562 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-03-21T12:27:20.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:20 smithi022 podman[24747]: 2024-03-21 12:27:20.138346241 +0000 UTC m=+1.272841494 container create 0d248775e0e517f2b9eb507a302cf35719effa905a33af734b742af906a3edff (image=quay.io/ceph/ceph:v17.2.0, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-mon-smithi022, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, name=centos-stream, version=8, vendor=Red Hat, Inc., ceph=True, GIT_CLEAN=True, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, distribution-scope=public, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, release=754, build-date=2022-05-03T08:36:31.336870) 2024-03-21T12:27:22.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 podman[24747]: 2024-03-21 12:27:21.713441181 +0000 UTC m=+2.847936461 container init 0d248775e0e517f2b9eb507a302cf35719effa905a33af734b742af906a3edff (image=quay.io/ceph/ceph:v17.2.0, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-mon-smithi022, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, vcs-type=git, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, distribution-scope=public, ceph=True, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, architecture=x86_64, GIT_CLEAN=True) 2024-03-21T12:27:22.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 podman[24747]: 2024-03-21 12:27:21.717354775 +0000 UTC m=+2.851850032 container start 0d248775e0e517f2b9eb507a302cf35719effa905a33af734b742af906a3edff (image=quay.io/ceph/ceph:v17.2.0, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-mon-smithi022, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, version=8, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., ceph=True, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, distribution-scope=public, io.openshift.expose-services=) 2024-03-21T12:27:22.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: set uid:gid to 167:167 (ceph:ceph) 2024-03-21T12:27:22.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-03-21T12:27:22.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: pidfile_write: ignore empty --pid-file 2024-03-21T12:27:22.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: load: jerasure load: lrc 2024-03-21T12:27:22.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: RocksDB version: 6.15.5 2024-03-21T12:27:22.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-03-21T12:27:22.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Compile date Apr 18 2022 2024-03-21T12:27:22.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: DB SUMMARY 2024-03-21T12:27:22.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: DB Session ID: CR7ZLCPJC7743FLS3R0U 2024-03-21T12:27:22.096 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: CURRENT file: CURRENT 2024-03-21T12:27:22.096 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: IDENTITY file: IDENTITY 2024-03-21T12:27:22.096 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-03-21T12:27:22.096 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi022/store.db dir, Total Num: 1, files: 000008.sst 2024-03-21T12:27:22.096 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi022/store.db: 000010.log size: 82711 ; 2024-03-21T12:27:22.096 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.error_if_exists: 0 2024-03-21T12:27:22.096 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.create_if_missing: 0 2024-03-21T12:27:22.096 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.paranoid_checks: 1 2024-03-21T12:27:22.096 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-03-21T12:27:22.096 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.env: 0x55566c2c6860 2024-03-21T12:27:22.096 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.fs: Posix File System 2024-03-21T12:27:22.096 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.info_log: 0x55566e169fa0 2024-03-21T12:27:22.096 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.max_file_opening_threads: 16 2024-03-21T12:27:22.096 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.statistics: (nil) 2024-03-21T12:27:22.096 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.use_fsync: 0 2024-03-21T12:27:22.096 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.max_log_file_size: 0 2024-03-21T12:27:22.097 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-03-21T12:27:22.097 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.log_file_time_to_roll: 0 2024-03-21T12:27:22.097 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.keep_log_file_num: 1000 2024-03-21T12:27:22.097 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.recycle_log_file_num: 0 2024-03-21T12:27:22.097 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.allow_fallocate: 1 2024-03-21T12:27:22.097 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.allow_mmap_reads: 0 2024-03-21T12:27:22.097 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.allow_mmap_writes: 0 2024-03-21T12:27:22.097 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.use_direct_reads: 0 2024-03-21T12:27:22.097 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-03-21T12:27:22.097 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.create_missing_column_families: 0 2024-03-21T12:27:22.097 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.db_log_dir: 2024-03-21T12:27:22.097 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-smithi022/store.db 2024-03-21T12:27:22.097 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.table_cache_numshardbits: 6 2024-03-21T12:27:22.097 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.WAL_ttl_seconds: 0 2024-03-21T12:27:22.097 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.WAL_size_limit_MB: 0 2024-03-21T12:27:22.098 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-03-21T12:27:22.098 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-03-21T12:27:22.098 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.is_fd_close_on_exec: 1 2024-03-21T12:27:22.098 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.advise_random_on_open: 1 2024-03-21T12:27:22.099 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.db_write_buffer_size: 0 2024-03-21T12:27:22.099 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.write_buffer_manager: 0x55566e25a330 2024-03-21T12:27:22.099 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-03-21T12:27:22.099 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-03-21T12:27:22.099 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-03-21T12:27:22.099 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.use_adaptive_mutex: 0 2024-03-21T12:27:22.099 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.rate_limiter: (nil) 2024-03-21T12:27:22.099 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-03-21T12:27:22.100 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.wal_recovery_mode: 2 2024-03-21T12:27:22.100 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.enable_thread_tracking: 0 2024-03-21T12:27:22.100 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.enable_pipelined_write: 0 2024-03-21T12:27:22.100 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.unordered_write: 0 2024-03-21T12:27:22.100 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-03-21T12:27:22.100 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-03-21T12:27:22.100 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-03-21T12:27:22.100 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-03-21T12:27:22.100 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.row_cache: None 2024-03-21T12:27:22.100 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.wal_filter: None 2024-03-21T12:27:22.100 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-03-21T12:27:22.100 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.allow_ingest_behind: 0 2024-03-21T12:27:22.100 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.preserve_deletes: 0 2024-03-21T12:27:22.100 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.two_write_queues: 0 2024-03-21T12:27:22.100 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.manual_wal_flush: 0 2024-03-21T12:27:22.100 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.atomic_flush: 0 2024-03-21T12:27:22.101 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-03-21T12:27:22.101 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.persist_stats_to_disk: 0 2024-03-21T12:27:22.101 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.write_dbid_to_manifest: 0 2024-03-21T12:27:22.101 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.log_readahead_size: 0 2024-03-21T12:27:22.101 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-03-21T12:27:22.101 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.best_efforts_recovery: 0 2024-03-21T12:27:22.101 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-03-21T12:27:22.101 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-03-21T12:27:22.101 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.allow_data_in_errors: 0 2024-03-21T12:27:22.101 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.db_host_id: __hostname__ 2024-03-21T12:27:22.101 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.max_background_jobs: 2 2024-03-21T12:27:22.101 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.max_background_compactions: -1 2024-03-21T12:27:22.101 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.max_subcompactions: 1 2024-03-21T12:27:22.101 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-03-21T12:27:22.101 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-03-21T12:27:22.102 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.delayed_write_rate : 16777216 2024-03-21T12:27:22.102 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.max_total_wal_size: 0 2024-03-21T12:27:22.102 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-03-21T12:27:22.102 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.stats_dump_period_sec: 600 2024-03-21T12:27:22.102 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.stats_persist_period_sec: 600 2024-03-21T12:27:22.102 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-03-21T12:27:22.102 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.max_open_files: -1 2024-03-21T12:27:22.102 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.bytes_per_sync: 0 2024-03-21T12:27:22.102 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.wal_bytes_per_sync: 0 2024-03-21T12:27:22.102 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.strict_bytes_per_sync: 0 2024-03-21T12:27:22.102 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.compaction_readahead_size: 0 2024-03-21T12:27:22.102 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.max_background_flushes: -1 2024-03-21T12:27:22.102 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Compression algorithms supported: 2024-03-21T12:27:22.102 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-03-21T12:27:22.102 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: kZSTD supported: 0 2024-03-21T12:27:22.103 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: kXpressCompression supported: 0 2024-03-21T12:27:22.103 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: kLZ4HCCompression supported: 1 2024-03-21T12:27:22.103 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: kLZ4Compression supported: 1 2024-03-21T12:27:22.103 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: kBZip2Compression supported: 0 2024-03-21T12:27:22.103 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: kZlibCompression supported: 1 2024-03-21T12:27:22.103 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: kSnappyCompression supported: 1 2024-03-21T12:27:22.103 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Fast CRC32 supported: Supported on x86 2024-03-21T12:27:22.103 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi022/store.db/MANIFEST-000009 2024-03-21T12:27:22.103 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-03-21T12:27:22.103 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-03-21T12:27:22.103 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.merge_operator: 2024-03-21T12:27:22.103 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.compaction_filter: None 2024-03-21T12:27:22.103 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.compaction_filter_factory: None 2024-03-21T12:27:22.103 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.sst_partitioner_factory: None 2024-03-21T12:27:22.103 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.memtable_factory: SkipListFactory 2024-03-21T12:27:22.104 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.table_factory: BlockBasedTable 2024-03-21T12:27:22.104 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55566e135d00) 2024-03-21T12:27:22.104 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: cache_index_and_filter_blocks: 1 2024-03-21T12:27:22.104 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-03-21T12:27:22.104 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-03-21T12:27:22.104 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: pin_top_level_index_and_filter: 1 2024-03-21T12:27:22.104 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: index_type: 0 2024-03-21T12:27:22.104 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: data_block_index_type: 0 2024-03-21T12:27:22.104 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: index_shortening: 1 2024-03-21T12:27:22.104 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: data_block_hash_table_util_ratio: 0.750000 2024-03-21T12:27:22.105 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: hash_index_allow_collision: 1 2024-03-21T12:27:22.105 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: checksum: 1 2024-03-21T12:27:22.105 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: no_block_cache: 0 2024-03-21T12:27:22.105 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: block_cache: 0x55566e1a0170 2024-03-21T12:27:22.105 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: block_cache_name: BinnedLRUCache 2024-03-21T12:27:22.105 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: block_cache_options: 2024-03-21T12:27:22.105 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: capacity : 536870912 2024-03-21T12:27:22.105 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: num_shard_bits : 4 2024-03-21T12:27:22.105 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: strict_capacity_limit : 0 2024-03-21T12:27:22.105 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: high_pri_pool_ratio: 0.000 2024-03-21T12:27:22.105 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: block_cache_compressed: (nil) 2024-03-21T12:27:22.105 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: persistent_cache: (nil) 2024-03-21T12:27:22.105 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: block_size: 4096 2024-03-21T12:27:22.106 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: block_size_deviation: 10 2024-03-21T12:27:22.106 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: block_restart_interval: 16 2024-03-21T12:27:22.106 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: index_block_restart_interval: 1 2024-03-21T12:27:22.106 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: metadata_block_size: 4096 2024-03-21T12:27:22.106 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: partition_filters: 0 2024-03-21T12:27:22.106 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: use_delta_encoding: 1 2024-03-21T12:27:22.106 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-03-21T12:27:22.106 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: whole_key_filtering: 1 2024-03-21T12:27:22.106 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: verify_compression: 0 2024-03-21T12:27:22.106 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: read_amp_bytes_per_bit: 0 2024-03-21T12:27:22.106 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: format_version: 4 2024-03-21T12:27:22.106 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: enable_index_compression: 1 2024-03-21T12:27:22.107 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: block_align: 0 2024-03-21T12:27:22.107 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.write_buffer_size: 33554432 2024-03-21T12:27:22.107 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.max_write_buffer_number: 2 2024-03-21T12:27:22.107 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.compression: NoCompression 2024-03-21T12:27:22.107 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.bottommost_compression: Disabled 2024-03-21T12:27:22.107 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.prefix_extractor: nullptr 2024-03-21T12:27:22.107 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-03-21T12:27:22.107 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.num_levels: 7 2024-03-21T12:27:22.107 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-03-21T12:27:22.107 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-03-21T12:27:22.107 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-03-21T12:27:22.107 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-03-21T12:27:22.107 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-03-21T12:27:22.107 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-03-21T12:27:22.108 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-03-21T12:27:22.108 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-03-21T12:27:22.108 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-03-21T12:27:22.108 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-03-21T12:27:22.108 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.compression_opts.window_bits: -14 2024-03-21T12:27:22.108 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.compression_opts.level: 32767 2024-03-21T12:27:22.108 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.compression_opts.strategy: 0 2024-03-21T12:27:22.108 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-03-21T12:27:22.109 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-03-21T12:27:22.109 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-03-21T12:27:22.109 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.compression_opts.enabled: false 2024-03-21T12:27:22.109 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-03-21T12:27:22.109 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-03-21T12:27:22.109 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-03-21T12:27:22.109 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.target_file_size_base: 67108864 2024-03-21T12:27:22.109 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.target_file_size_multiplier: 1 2024-03-21T12:27:22.109 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-03-21T12:27:22.109 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-03-21T12:27:22.109 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-03-21T12:27:22.109 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-03-21T12:27:22.109 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-03-21T12:27:22.109 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-03-21T12:27:22.110 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-03-21T12:27:22.110 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-03-21T12:27:22.110 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-03-21T12:27:22.110 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-03-21T12:27:22.110 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-03-21T12:27:22.110 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-03-21T12:27:22.110 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.arena_block_size: 4194304 2024-03-21T12:27:22.110 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-03-21T12:27:22.110 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-03-21T12:27:22.111 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-03-21T12:27:22.111 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.disable_auto_compactions: 0 2024-03-21T12:27:22.111 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-03-21T12:27:22.111 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-03-21T12:27:22.111 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-03-21T12:27:22.111 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-03-21T12:27:22.111 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-03-21T12:27:22.111 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-03-21T12:27:22.111 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-03-21T12:27:22.111 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-03-21T12:27:22.111 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-03-21T12:27:22.111 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-03-21T12:27:22.111 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.table_properties_collectors: 2024-03-21T12:27:22.112 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.inplace_update_support: 0 2024-03-21T12:27:22.112 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.inplace_update_num_locks: 10000 2024-03-21T12:27:22.112 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-03-21T12:27:22.112 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-03-21T12:27:22.112 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.memtable_huge_page_size: 0 2024-03-21T12:27:22.112 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.bloom_locality: 0 2024-03-21T12:27:22.112 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.max_successive_merges: 0 2024-03-21T12:27:22.112 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.optimize_filters_for_hits: 0 2024-03-21T12:27:22.112 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.paranoid_file_checks: 0 2024-03-21T12:27:22.113 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.force_consistency_checks: 1 2024-03-21T12:27:22.113 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.report_bg_io_stats: 0 2024-03-21T12:27:22.113 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.ttl: 2592000 2024-03-21T12:27:22.113 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.periodic_compaction_seconds: 0 2024-03-21T12:27:22.113 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.enable_blob_files: false 2024-03-21T12:27:22.113 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.min_blob_size: 0 2024-03-21T12:27:22.113 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.blob_file_size: 268435456 2024-03-21T12:27:22.113 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.blob_compression_type: NoCompression 2024-03-21T12:27:22.113 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.enable_blob_garbage_collection: false 2024-03-21T12:27:22.113 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-03-21T12:27:22.114 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi022/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-03-21T12:27:22.114 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 5 2024-03-21T12:27:22.114 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: [db/version_set.cc:4083] Creating manifest 13 2024-03-21T12:27:22.114 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: EVENT_LOG_v1 {"time_micros": 1711024041747613, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-03-21T12:27:22.114 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #10 mode 2 2024-03-21T12:27:22.114 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-03-21T12:27:22.114 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: EVENT_LOG_v1 {"time_micros": 1711024041748456, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 79393, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 77657, "index_size": 224, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 9884, "raw_average_key_size": 47, "raw_value_size": 71966, "raw_average_value_size": 344, "num_data_blocks": 10, "num_entries": 209, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1711024041, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "e89af291-bcac-42d2-8583-b4ba1c2dea86", "db_session_id": "CR7ZLCPJC7743FLS3R0U"}} 2024-03-21T12:27:22.114 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: [db/version_set.cc:4083] Creating manifest 15 2024-03-21T12:27:22.114 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: EVENT_LOG_v1 {"time_micros": 1711024041748868, "job": 1, "event": "recovery_finished"} 2024-03-21T12:27:22.114 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-smithi022/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-03-21T12:27:22.115 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x55566e186a80 2024-03-21T12:27:22.115 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: DB pointer 0x55566e196000 2024-03-21T12:27:22.115 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-03-21T12:27:22.115 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-03-21T12:27:22.115 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: ** DB Stats ** 2024-03-21T12:27:22.115 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-03-21T12:27:22.115 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-03-21T12:27:22.115 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-03-21T12:27:22.115 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-03-21T12:27:22.115 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-03-21T12:27:22.115 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-03-21T12:27:22.115 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-03-21T12:27:22.115 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: 2024-03-21T12:27:22.115 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: ** Compaction Stats [default] ** 2024-03-21T12:27:22.115 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-03-21T12:27:22.116 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-03-21T12:27:22.116 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: L0 2/0 79.33 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 130.1 0.00 0.00 1 0.001 0 0 2024-03-21T12:27:22.116 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: Sum 2/0 79.33 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 130.1 0.00 0.00 1 0.001 0 0 2024-03-21T12:27:22.116 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 130.1 0.00 0.00 1 0.001 0 0 2024-03-21T12:27:22.116 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: 2024-03-21T12:27:22.116 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: ** Compaction Stats [default] ** 2024-03-21T12:27:22.116 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-03-21T12:27:22.116 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-03-21T12:27:22.116 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 130.1 0.00 0.00 1 0.001 0 0 2024-03-21T12:27:22.116 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-03-21T12:27:22.116 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-03-21T12:27:22.116 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-03-21T12:27:22.116 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-03-21T12:27:22.116 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-03-21T12:27:22.116 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: AddFile(Keys): cumulative 0, interval 0 2024-03-21T12:27:22.117 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: Cumulative compaction: 0.00 GB write, 21.44 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-21T12:27:22.117 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: Interval compaction: 0.00 GB write, 21.44 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-21T12:27:22.117 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-03-21T12:27:22.117 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: 2024-03-21T12:27:22.117 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: ** File Read Latency Histogram By Level [default] ** 2024-03-21T12:27:22.117 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: 2024-03-21T12:27:22.117 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: ** Compaction Stats [default] ** 2024-03-21T12:27:22.117 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-03-21T12:27:22.117 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-03-21T12:27:22.117 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: L0 2/0 79.33 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 130.1 0.00 0.00 1 0.001 0 0 2024-03-21T12:27:22.117 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: Sum 2/0 79.33 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 130.1 0.00 0.00 1 0.001 0 0 2024-03-21T12:27:22.117 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-03-21T12:27:22.117 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: 2024-03-21T12:27:22.117 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: ** Compaction Stats [default] ** 2024-03-21T12:27:22.117 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-03-21T12:27:22.118 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-03-21T12:27:22.118 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 130.1 0.00 0.00 1 0.001 0 0 2024-03-21T12:27:22.118 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-03-21T12:27:22.118 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-03-21T12:27:22.118 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-03-21T12:27:22.118 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-03-21T12:27:22.118 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-03-21T12:27:22.118 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: AddFile(Keys): cumulative 0, interval 0 2024-03-21T12:27:22.118 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: Cumulative compaction: 0.00 GB write, 21.24 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-21T12:27:22.118 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-21T12:27:22.118 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-03-21T12:27:22.119 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: 2024-03-21T12:27:22.119 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: ** File Read Latency Histogram By Level [default] ** 2024-03-21T12:27:22.119 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: starting mon.smithi022 rank 0 at public addrs [v2:172.21.15.22:3300/0,v1:172.21.15.22:6789/0] at bind addrs [v2:172.21.15.22:3300/0,v1:172.21.15.22:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi022 fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 2024-03-21T12:27:22.119 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: mon.smithi022@-1(???) e1 preinit fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 2024-03-21T12:27:22.119 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: mon.smithi022@-1(???).mds e1 new map 2024-03-21T12:27:22.119 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: mon.smithi022@-1(???).mds e1 print_map 2024-03-21T12:27:22.119 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: e1 2024-03-21T12:27:22.119 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-03-21T12:27:22.119 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-03-21T12:27:22.119 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: legacy client fscid: -1 2024-03-21T12:27:22.119 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: 2024-03-21T12:27:22.119 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: No filesystems configured 2024-03-21T12:27:22.119 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: mon.smithi022@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-03-21T12:27:22.119 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: mon.smithi022@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-03-21T12:27:22.120 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: mon.smithi022@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-03-21T12:27:22.120 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: mon.smithi022@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-03-21T12:27:22.120 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: mon.smithi022@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-03-21T12:27:22.120 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: expand_channel_meta expand map: {default=false} 2024-03-21T12:27:22.120 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: expand_channel_meta from 'false' to 'false' 2024-03-21T12:27:22.120 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: expand_channel_meta expanded map: {default=false} 2024-03-21T12:27:22.120 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: expand_channel_meta expand map: {default=info} 2024-03-21T12:27:22.120 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: expand_channel_meta from 'info' to 'info' 2024-03-21T12:27:22.120 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: expand_channel_meta expanded map: {default=info} 2024-03-21T12:27:22.120 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: expand_channel_meta expand map: {default=daemon} 2024-03-21T12:27:22.120 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: expand_channel_meta from 'daemon' to 'daemon' 2024-03-21T12:27:22.120 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: expand_channel_meta expanded map: {default=daemon} 2024-03-21T12:27:22.120 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: expand_channel_meta expand map: {default=debug} 2024-03-21T12:27:22.121 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: expand_channel_meta from 'debug' to 'debug' 2024-03-21T12:27:22.121 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:21 smithi022 ceph-mon[24777]: expand_channel_meta expanded map: {default=debug} 2024-03-21T12:27:22.451 INFO:teuthology.orchestra.run.smithi022.stderr:Setting mon public_network to 172.21.0.0/20 2024-03-21T12:27:22.738 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:22 smithi022 bash[24747]: 0d248775e0e517f2b9eb507a302cf35719effa905a33af734b742af906a3edff 2024-03-21T12:27:22.739 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:22 smithi022 systemd[1]: Started Ceph mon.smithi022 for 3fc7f0f2-e77e-11ee-95cd-87774f69a715. 2024-03-21T12:27:23.032 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:22 smithi022 ceph-mon[24777]: mon.smithi022 is new leader, mons smithi022 in quorum (ranks 0) 2024-03-21T12:27:23.032 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:22 smithi022 ceph-mon[24777]: monmap e1: 1 mons at {smithi022=[v2:172.21.15.22:3300/0,v1:172.21.15.22:6789/0]} 2024-03-21T12:27:23.032 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:22 smithi022 ceph-mon[24777]: fsmap 2024-03-21T12:27:23.033 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:22 smithi022 ceph-mon[24777]: osdmap e1: 0 total, 0 up, 0 in 2024-03-21T12:27:23.033 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:22 smithi022 ceph-mon[24777]: mgrmap e1: no daemons active 2024-03-21T12:27:23.639 INFO:teuthology.orchestra.run.smithi022.stderr:Wrote config to /etc/ceph/ceph.conf 2024-03-21T12:27:23.639 INFO:teuthology.orchestra.run.smithi022.stderr:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-03-21T12:27:23.639 INFO:teuthology.orchestra.run.smithi022.stderr:Creating mgr... 2024-03-21T12:27:23.639 INFO:teuthology.orchestra.run.smithi022.stderr:Verifying port 9283 ... 2024-03-21T12:27:23.802 INFO:teuthology.orchestra.run.smithi022.stderr:systemctl: Failed to reset failed state of unit ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715@mgr.smithi022.fektzs.service: Unit ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715@mgr.smithi022.fektzs.service not loaded. 2024-03-21T12:27:23.809 INFO:teuthology.orchestra.run.smithi022.stderr:systemctl: Created symlink /etc/systemd/system/ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715.target.wants/ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715@mgr.smithi022.fektzs.service → /etc/systemd/system/ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715@.service. 2024-03-21T12:27:24.284 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:24 smithi022 ceph-mon[24777]: from='client.? 172.21.15.22:0/1878823033' entity='client.admin' 2024-03-21T12:27:24.478 INFO:teuthology.orchestra.run.smithi022.stderr:systemctl: disabled 2024-03-21T12:27:24.486 INFO:teuthology.orchestra.run.smithi022.stderr:systemctl: inactive 2024-03-21T12:27:24.487 INFO:teuthology.orchestra.run.smithi022.stderr:firewalld.service is not enabled 2024-03-21T12:27:24.487 INFO:teuthology.orchestra.run.smithi022.stderr:Not possible to enable service . firewalld.service is not available 2024-03-21T12:27:24.496 INFO:teuthology.orchestra.run.smithi022.stderr:systemctl: disabled 2024-03-21T12:27:24.503 INFO:teuthology.orchestra.run.smithi022.stderr:systemctl: inactive 2024-03-21T12:27:24.504 INFO:teuthology.orchestra.run.smithi022.stderr:firewalld.service is not enabled 2024-03-21T12:27:24.504 INFO:teuthology.orchestra.run.smithi022.stderr:Not possible to open ports <[9283]>. firewalld.service is not available 2024-03-21T12:27:24.504 INFO:teuthology.orchestra.run.smithi022.stderr:Waiting for mgr to start... 2024-03-21T12:27:24.504 INFO:teuthology.orchestra.run.smithi022.stderr:Waiting for mgr... 2024-03-21T12:27:24.978 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: 2024-03-21T12:27:24.978 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: { 2024-03-21T12:27:24.978 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "fsid": "3fc7f0f2-e77e-11ee-95cd-87774f69a715", 2024-03-21T12:27:24.978 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "health": { 2024-03-21T12:27:24.978 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-03-21T12:27:24.979 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "checks": {}, 2024-03-21T12:27:24.979 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "mutes": [] 2024-03-21T12:27:24.979 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: }, 2024-03-21T12:27:24.979 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-03-21T12:27:24.979 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "quorum": [ 2024-03-21T12:27:24.979 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: 0 2024-03-21T12:27:24.979 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: ], 2024-03-21T12:27:24.979 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "quorum_names": [ 2024-03-21T12:27:24.979 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "smithi022" 2024-03-21T12:27:24.979 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: ], 2024-03-21T12:27:24.979 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "quorum_age": 3, 2024-03-21T12:27:24.979 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "monmap": { 2024-03-21T12:27:24.979 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "epoch": 1, 2024-03-21T12:27:24.979 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-03-21T12:27:24.979 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "num_mons": 1 2024-03-21T12:27:24.980 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: }, 2024-03-21T12:27:24.980 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "osdmap": { 2024-03-21T12:27:24.980 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "epoch": 1, 2024-03-21T12:27:24.980 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "num_osds": 0, 2024-03-21T12:27:24.980 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-03-21T12:27:24.980 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-03-21T12:27:24.980 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-03-21T12:27:24.980 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-03-21T12:27:24.980 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-03-21T12:27:24.980 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: }, 2024-03-21T12:27:24.980 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "pgmap": { 2024-03-21T12:27:24.980 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-03-21T12:27:24.980 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-03-21T12:27:24.980 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "num_pools": 0, 2024-03-21T12:27:24.980 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "num_objects": 0, 2024-03-21T12:27:24.981 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-03-21T12:27:24.981 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-03-21T12:27:24.981 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-03-21T12:27:24.981 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "bytes_total": 0 2024-03-21T12:27:24.981 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: }, 2024-03-21T12:27:24.981 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "fsmap": { 2024-03-21T12:27:24.981 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "epoch": 1, 2024-03-21T12:27:24.981 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "by_rank": [], 2024-03-21T12:27:24.981 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "up:standby": 0 2024-03-21T12:27:24.981 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: }, 2024-03-21T12:27:24.981 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "mgrmap": { 2024-03-21T12:27:24.981 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "available": false, 2024-03-21T12:27:24.981 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-03-21T12:27:24.981 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "modules": [ 2024-03-21T12:27:24.981 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "iostat", 2024-03-21T12:27:24.981 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "nfs", 2024-03-21T12:27:24.982 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "restful" 2024-03-21T12:27:24.982 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: ], 2024-03-21T12:27:24.982 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "services": {} 2024-03-21T12:27:24.982 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: }, 2024-03-21T12:27:24.982 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "servicemap": { 2024-03-21T12:27:24.982 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "epoch": 1, 2024-03-21T12:27:24.982 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "modified": "2024-03-21T12:27:08.895633+0000", 2024-03-21T12:27:24.982 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "services": {} 2024-03-21T12:27:24.982 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: }, 2024-03-21T12:27:24.982 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "progress_events": {} 2024-03-21T12:27:24.982 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: } 2024-03-21T12:27:25.588 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:25 smithi022 ceph-mon[24777]: from='client.? 172.21.15.22:0/1038800414' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-03-21T12:27:25.598 INFO:teuthology.orchestra.run.smithi022.stderr:mgr not available, waiting (1/15)... 2024-03-21T12:27:28.052 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: 2024-03-21T12:27:28.052 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: { 2024-03-21T12:27:28.052 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "fsid": "3fc7f0f2-e77e-11ee-95cd-87774f69a715", 2024-03-21T12:27:28.052 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "health": { 2024-03-21T12:27:28.052 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-03-21T12:27:28.052 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "checks": {}, 2024-03-21T12:27:28.052 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "mutes": [] 2024-03-21T12:27:28.052 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: }, 2024-03-21T12:27:28.052 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-03-21T12:27:28.053 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "quorum": [ 2024-03-21T12:27:28.053 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: 0 2024-03-21T12:27:28.053 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: ], 2024-03-21T12:27:28.053 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "quorum_names": [ 2024-03-21T12:27:28.053 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "smithi022" 2024-03-21T12:27:28.053 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: ], 2024-03-21T12:27:28.053 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "quorum_age": 6, 2024-03-21T12:27:28.053 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "monmap": { 2024-03-21T12:27:28.053 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "epoch": 1, 2024-03-21T12:27:28.053 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-03-21T12:27:28.053 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "num_mons": 1 2024-03-21T12:27:28.053 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: }, 2024-03-21T12:27:28.053 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "osdmap": { 2024-03-21T12:27:28.053 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "epoch": 1, 2024-03-21T12:27:28.053 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "num_osds": 0, 2024-03-21T12:27:28.053 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-03-21T12:27:28.054 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-03-21T12:27:28.054 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-03-21T12:27:28.057 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-03-21T12:27:28.057 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-03-21T12:27:28.057 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: }, 2024-03-21T12:27:28.057 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "pgmap": { 2024-03-21T12:27:28.057 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-03-21T12:27:28.058 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-03-21T12:27:28.058 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "num_pools": 0, 2024-03-21T12:27:28.058 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "num_objects": 0, 2024-03-21T12:27:28.058 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-03-21T12:27:28.058 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-03-21T12:27:28.058 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-03-21T12:27:28.058 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "bytes_total": 0 2024-03-21T12:27:28.058 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: }, 2024-03-21T12:27:28.058 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "fsmap": { 2024-03-21T12:27:28.058 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "epoch": 1, 2024-03-21T12:27:28.058 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "by_rank": [], 2024-03-21T12:27:28.058 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "up:standby": 0 2024-03-21T12:27:28.058 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: }, 2024-03-21T12:27:28.058 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "mgrmap": { 2024-03-21T12:27:28.058 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "available": false, 2024-03-21T12:27:28.059 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-03-21T12:27:28.059 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "modules": [ 2024-03-21T12:27:28.059 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "iostat", 2024-03-21T12:27:28.059 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "nfs", 2024-03-21T12:27:28.059 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "restful" 2024-03-21T12:27:28.059 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: ], 2024-03-21T12:27:28.059 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "services": {} 2024-03-21T12:27:28.059 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: }, 2024-03-21T12:27:28.059 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "servicemap": { 2024-03-21T12:27:28.059 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "epoch": 1, 2024-03-21T12:27:28.059 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "modified": "2024-03-21T12:27:08.895633+0000", 2024-03-21T12:27:28.059 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "services": {} 2024-03-21T12:27:28.059 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: }, 2024-03-21T12:27:28.059 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "progress_events": {} 2024-03-21T12:27:28.059 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: } 2024-03-21T12:27:28.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:28 smithi022 ceph-mon[24777]: from='client.? 172.21.15.22:0/710736237' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-03-21T12:27:30.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:30 smithi022 ceph-mon[24777]: Activating manager daemon smithi022.fektzs 2024-03-21T12:27:30.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:30 smithi022 ceph-mon[24777]: mgrmap e2: smithi022.fektzs(active, starting, since 0.0027739s) 2024-03-21T12:27:30.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:30 smithi022 ceph-mon[24777]: from='mgr.14100 172.21.15.22:0/3264457355' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-21T12:27:30.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:30 smithi022 ceph-mon[24777]: from='mgr.14100 172.21.15.22:0/3264457355' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-21T12:27:30.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:30 smithi022 ceph-mon[24777]: from='mgr.14100 172.21.15.22:0/3264457355' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-21T12:27:30.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:30 smithi022 ceph-mon[24777]: from='mgr.14100 172.21.15.22:0/3264457355' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mon metadata", "id": "smithi022"}]: dispatch 2024-03-21T12:27:30.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:30 smithi022 ceph-mon[24777]: from='mgr.14100 172.21.15.22:0/3264457355' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mgr metadata", "who": "smithi022.fektzs", "id": "smithi022.fektzs"}]: dispatch 2024-03-21T12:27:30.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:30 smithi022 ceph-mon[24777]: Manager daemon smithi022.fektzs is now available 2024-03-21T12:27:30.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:30 smithi022 ceph-mon[24777]: from='mgr.14100 172.21.15.22:0/3264457355' entity='mgr.smithi022.fektzs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi022.fektzs/mirror_snapshot_schedule"}]: dispatch 2024-03-21T12:27:30.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:30 smithi022 ceph-mon[24777]: from='mgr.14100 172.21.15.22:0/3264457355' entity='mgr.smithi022.fektzs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi022.fektzs/trash_purge_schedule"}]: dispatch 2024-03-21T12:27:30.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:30 smithi022 ceph-mon[24777]: from='mgr.14100 172.21.15.22:0/3264457355' entity='mgr.smithi022.fektzs' 2024-03-21T12:27:30.846 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:30 smithi022 ceph-mon[24777]: from='mgr.14100 172.21.15.22:0/3264457355' entity='mgr.smithi022.fektzs' 2024-03-21T12:27:30.846 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:30 smithi022 ceph-mon[24777]: from='mgr.14100 172.21.15.22:0/3264457355' entity='mgr.smithi022.fektzs' 2024-03-21T12:27:30.849 INFO:teuthology.orchestra.run.smithi022.stderr:mgr not available, waiting (2/15)... 2024-03-21T12:27:32.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:32 smithi022 ceph-mon[24777]: mgrmap e3: smithi022.fektzs(active, since 1.00703s) 2024-03-21T12:27:33.559 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: 2024-03-21T12:27:33.559 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: { 2024-03-21T12:27:33.559 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "fsid": "3fc7f0f2-e77e-11ee-95cd-87774f69a715", 2024-03-21T12:27:33.559 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "health": { 2024-03-21T12:27:33.559 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-03-21T12:27:33.559 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "checks": {}, 2024-03-21T12:27:33.559 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "mutes": [] 2024-03-21T12:27:33.560 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: }, 2024-03-21T12:27:33.560 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-03-21T12:27:33.560 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "quorum": [ 2024-03-21T12:27:33.560 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: 0 2024-03-21T12:27:33.560 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: ], 2024-03-21T12:27:33.560 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "quorum_names": [ 2024-03-21T12:27:33.560 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "smithi022" 2024-03-21T12:27:33.560 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: ], 2024-03-21T12:27:33.560 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "quorum_age": 11, 2024-03-21T12:27:33.560 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "monmap": { 2024-03-21T12:27:33.560 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "epoch": 1, 2024-03-21T12:27:33.560 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-03-21T12:27:33.561 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "num_mons": 1 2024-03-21T12:27:33.561 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: }, 2024-03-21T12:27:33.561 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "osdmap": { 2024-03-21T12:27:33.561 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "epoch": 1, 2024-03-21T12:27:33.561 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "num_osds": 0, 2024-03-21T12:27:33.561 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-03-21T12:27:33.561 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-03-21T12:27:33.565 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-03-21T12:27:33.565 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-03-21T12:27:33.565 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-03-21T12:27:33.565 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: }, 2024-03-21T12:27:33.565 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "pgmap": { 2024-03-21T12:27:33.565 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-03-21T12:27:33.565 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-03-21T12:27:33.565 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "num_pools": 0, 2024-03-21T12:27:33.565 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "num_objects": 0, 2024-03-21T12:27:33.566 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-03-21T12:27:33.566 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-03-21T12:27:33.566 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-03-21T12:27:33.566 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "bytes_total": 0 2024-03-21T12:27:33.566 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: }, 2024-03-21T12:27:33.566 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "fsmap": { 2024-03-21T12:27:33.566 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "epoch": 1, 2024-03-21T12:27:33.566 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "by_rank": [], 2024-03-21T12:27:33.566 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "up:standby": 0 2024-03-21T12:27:33.566 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: }, 2024-03-21T12:27:33.566 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "mgrmap": { 2024-03-21T12:27:33.566 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "available": true, 2024-03-21T12:27:33.566 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-03-21T12:27:33.567 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "modules": [ 2024-03-21T12:27:33.567 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "iostat", 2024-03-21T12:27:33.567 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "nfs", 2024-03-21T12:27:33.567 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "restful" 2024-03-21T12:27:33.567 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: ], 2024-03-21T12:27:33.567 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "services": {} 2024-03-21T12:27:33.567 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: }, 2024-03-21T12:27:33.567 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "servicemap": { 2024-03-21T12:27:33.567 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "epoch": 1, 2024-03-21T12:27:33.567 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "modified": "2024-03-21T12:27:08.895633+0000", 2024-03-21T12:27:33.567 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "services": {} 2024-03-21T12:27:33.567 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: }, 2024-03-21T12:27:33.568 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "progress_events": {} 2024-03-21T12:27:33.568 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: } 2024-03-21T12:27:34.126 INFO:teuthology.orchestra.run.smithi022.stderr:mgr is available 2024-03-21T12:27:34.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:34 smithi022 ceph-mon[24777]: mgrmap e4: smithi022.fektzs(active, since 3s) 2024-03-21T12:27:34.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:34 smithi022 ceph-mon[24777]: from='client.? 172.21.15.22:0/2725876996' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-03-21T12:27:35.206 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: 2024-03-21T12:27:35.206 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: [global] 2024-03-21T12:27:35.206 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: fsid = 3fc7f0f2-e77e-11ee-95cd-87774f69a715 2024-03-21T12:27:35.206 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-03-21T12:27:35.207 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-03-21T12:27:35.207 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-03-21T12:27:35.207 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-03-21T12:27:35.207 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: 2024-03-21T12:27:35.207 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: [mgr] 2024-03-21T12:27:35.207 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-03-21T12:27:35.207 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: 2024-03-21T12:27:35.207 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: [osd] 2024-03-21T12:27:35.207 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-03-21T12:27:35.207 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-03-21T12:27:35.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:35 smithi022 ceph-mon[24777]: from='client.? 172.21.15.22:0/3450724895' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-03-21T12:27:35.863 INFO:teuthology.orchestra.run.smithi022.stderr:Enabling cephadm module... 2024-03-21T12:27:36.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:36 smithi022 ceph-mon[24777]: from='client.? 172.21.15.22:0/3297806293' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-03-21T12:27:37.753 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:37 smithi022 ceph-mon[24777]: from='client.? 172.21.15.22:0/3297806293' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-03-21T12:27:37.753 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:37 smithi022 ceph-mon[24777]: mgrmap e5: smithi022.fektzs(active, since 6s) 2024-03-21T12:27:37.764 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: { 2024-03-21T12:27:37.764 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "epoch": 5, 2024-03-21T12:27:37.764 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "available": true, 2024-03-21T12:27:37.764 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "active_name": "smithi022.fektzs", 2024-03-21T12:27:37.764 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "num_standby": 0 2024-03-21T12:27:37.764 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: } 2024-03-21T12:27:38.400 INFO:teuthology.orchestra.run.smithi022.stderr:Waiting for the mgr to restart... 2024-03-21T12:27:38.400 INFO:teuthology.orchestra.run.smithi022.stderr:Waiting for mgr epoch 5... 2024-03-21T12:27:38.623 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:38 smithi022 ceph-mon[24777]: from='client.? 172.21.15.22:0/2021318781' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-03-21T12:27:43.772 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: { 2024-03-21T12:27:43.772 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "mgrmap_epoch": 7, 2024-03-21T12:27:43.773 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "initialized": true 2024-03-21T12:27:43.773 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: } 2024-03-21T12:27:43.785 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:43 smithi022 ceph-mon[24777]: Active manager daemon smithi022.fektzs restarted 2024-03-21T12:27:43.785 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:43 smithi022 ceph-mon[24777]: Activating manager daemon smithi022.fektzs 2024-03-21T12:27:43.785 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:43 smithi022 ceph-mon[24777]: osdmap e2: 0 total, 0 up, 0 in 2024-03-21T12:27:43.785 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:43 smithi022 ceph-mon[24777]: mgrmap e6: smithi022.fektzs(active, starting, since 0.22918s) 2024-03-21T12:27:43.785 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:43 smithi022 ceph-mon[24777]: from='mgr.14118 172.21.15.22:0/380713134' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mon metadata", "id": "smithi022"}]: dispatch 2024-03-21T12:27:43.785 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:43 smithi022 ceph-mon[24777]: from='mgr.14118 172.21.15.22:0/380713134' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mgr metadata", "who": "smithi022.fektzs", "id": "smithi022.fektzs"}]: dispatch 2024-03-21T12:27:43.785 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:43 smithi022 ceph-mon[24777]: from='mgr.14118 172.21.15.22:0/380713134' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-21T12:27:43.785 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:43 smithi022 ceph-mon[24777]: from='mgr.14118 172.21.15.22:0/380713134' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-21T12:27:43.785 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:43 smithi022 ceph-mon[24777]: from='mgr.14118 172.21.15.22:0/380713134' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-21T12:27:43.785 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:43 smithi022 ceph-mon[24777]: Manager daemon smithi022.fektzs is now available 2024-03-21T12:27:43.785 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:43 smithi022 ceph-mon[24777]: from='mgr.14118 172.21.15.22:0/380713134' entity='mgr.smithi022.fektzs' 2024-03-21T12:27:43.785 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:43 smithi022 ceph-mon[24777]: from='mgr.14118 172.21.15.22:0/380713134' entity='mgr.smithi022.fektzs' 2024-03-21T12:27:43.786 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:43 smithi022 ceph-mon[24777]: from='mgr.14118 172.21.15.22:0/380713134' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:27:43.786 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:43 smithi022 ceph-mon[24777]: from='mgr.14118 172.21.15.22:0/380713134' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:27:43.786 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:43 smithi022 ceph-mon[24777]: from='mgr.14118 172.21.15.22:0/380713134' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:27:43.786 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:43 smithi022 ceph-mon[24777]: from='mgr.14118 172.21.15.22:0/380713134' entity='mgr.smithi022.fektzs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi022.fektzs/mirror_snapshot_schedule"}]: dispatch 2024-03-21T12:27:43.786 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:43 smithi022 ceph-mon[24777]: from='mgr.14118 172.21.15.22:0/380713134' entity='mgr.smithi022.fektzs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi022.fektzs/trash_purge_schedule"}]: dispatch 2024-03-21T12:27:44.280 INFO:teuthology.orchestra.run.smithi022.stderr:mgr epoch 5 is available 2024-03-21T12:27:44.280 INFO:teuthology.orchestra.run.smithi022.stderr:Setting orchestrator backend to cephadm... 2024-03-21T12:27:45.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:44 smithi022 ceph-mon[24777]: mgrmap e7: smithi022.fektzs(active, since 1.23129s) 2024-03-21T12:27:45.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:44 smithi022 ceph-mon[24777]: [21/Mar/2024:12:27:44] ENGINE Bus STARTING 2024-03-21T12:27:45.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:44 smithi022 ceph-mon[24777]: [21/Mar/2024:12:27:44] ENGINE Serving on https://172.21.15.22:7150 2024-03-21T12:27:45.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:44 smithi022 ceph-mon[24777]: [21/Mar/2024:12:27:44] ENGINE Bus STARTED 2024-03-21T12:27:45.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:44 smithi022 ceph-mon[24777]: from='mgr.14118 172.21.15.22:0/380713134' entity='mgr.smithi022.fektzs' 2024-03-21T12:27:45.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:44 smithi022 ceph-mon[24777]: from='mgr.14118 172.21.15.22:0/380713134' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:27:46.025 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:45 smithi022 ceph-mon[24777]: from='client.14130 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:27:46.025 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:45 smithi022 ceph-mon[24777]: from='mgr.14118 172.21.15.22:0/380713134' entity='mgr.smithi022.fektzs' 2024-03-21T12:27:46.025 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:45 smithi022 ceph-mon[24777]: from='mgr.14118 172.21.15.22:0/380713134' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:27:46.025 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:45 smithi022 ceph-mon[24777]: mgrmap e8: smithi022.fektzs(active, since 2s) 2024-03-21T12:27:46.026 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: value unchanged 2024-03-21T12:27:46.601 INFO:teuthology.orchestra.run.smithi022.stderr:Generating ssh key... 2024-03-21T12:27:47.101 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:46 smithi022 ceph-mon[24777]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:27:48.681 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:48 smithi022 ceph-mon[24777]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:27:48.681 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:48 smithi022 ceph-mon[24777]: Generating ssh key... 2024-03-21T12:27:48.682 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:48 smithi022 ceph-mon[24777]: from='mgr.14118 172.21.15.22:0/380713134' entity='mgr.smithi022.fektzs' 2024-03-21T12:27:48.682 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:48 smithi022 ceph-mon[24777]: from='mgr.14118 172.21.15.22:0/380713134' entity='mgr.smithi022.fektzs' 2024-03-21T12:27:48.682 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDLbGnL+TM5FZ0aSz9hpkXB5DREeXTvOMWfa1snuqJwRNjvTwRSricaUWlot6BZHLt9McNb6BXfNpjTt/BB+XzZ07W6X6NS/XmaNL33eEBvrjWNu6WCmUtgKCIHdY22hZQiOMSmNlKchvKQY0EA5resbSft8gKocfBNjs5Y53wAW9cYHhTrce4UgvV4U5rz80Xjd151VE2m69eA6hNtRSZ9tEDcRGeQZPEiyGDvY/F1nxsSdQGCG+QbhhKNpBHrRpiXWOL9z4KVmV5AHmnMQ4ZgBg82jeDMyHCwsIbW2lbsmL7jnB/ew+2Ymw4ehLkwWxknh9U0fCES70wt56861k1SPSQMBeOdz5JxhGzIIb/E1Zbj4lx7ofSJe3YTXjm88Sbdfdp9b5s/XBCfrRgrssHPZ78V6JhwmQ8a3SPGkFYD/TnJmYvrJkmE6MIv15cHyy3VKc27Le2bswrCI1KoP8TKMZFh++7BL6QYrZgxP3X1TV3/F+zfP4jOOyF48hjx1i0= ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715 2024-03-21T12:27:49.281 INFO:teuthology.orchestra.run.smithi022.stderr:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-03-21T12:27:49.281 INFO:teuthology.orchestra.run.smithi022.stderr:Adding key to root@localhost authorized_keys... 2024-03-21T12:27:49.281 INFO:teuthology.orchestra.run.smithi022.stderr:Adding host smithi022... 2024-03-21T12:27:49.663 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:49 smithi022 ceph-mon[24777]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:27:50.889 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:50 smithi022 ceph-mon[24777]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi022", "addr": "172.21.15.22", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:27:50.889 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:50 smithi022 ceph-mon[24777]: Deploying cephadm binary to smithi022 2024-03-21T12:27:50.890 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: Added host 'smithi022' with addr '172.21.15.22' 2024-03-21T12:27:51.495 INFO:teuthology.orchestra.run.smithi022.stderr:Deploying mon service with default placement... 2024-03-21T12:27:52.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:51 smithi022 ceph-mon[24777]: from='mgr.14118 172.21.15.22:0/380713134' entity='mgr.smithi022.fektzs' 2024-03-21T12:27:52.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:51 smithi022 ceph-mon[24777]: Added host smithi022 2024-03-21T12:27:52.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:51 smithi022 ceph-mon[24777]: from='mgr.14118 172.21.15.22:0/380713134' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:27:52.115 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: Scheduled mon update... 2024-03-21T12:27:52.810 INFO:teuthology.orchestra.run.smithi022.stderr:Deploying mgr service with default placement... 2024-03-21T12:27:53.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:53 smithi022 ceph-mon[24777]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:27:53.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:53 smithi022 ceph-mon[24777]: Saving service mon spec with placement count:5 2024-03-21T12:27:53.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:53 smithi022 ceph-mon[24777]: from='mgr.14118 172.21.15.22:0/380713134' entity='mgr.smithi022.fektzs' 2024-03-21T12:27:53.566 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: Scheduled mgr update... 2024-03-21T12:27:54.221 INFO:teuthology.orchestra.run.smithi022.stderr:Deploying crash service with default placement... 2024-03-21T12:27:54.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:54 smithi022 ceph-mon[24777]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:27:54.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:54 smithi022 ceph-mon[24777]: Saving service mgr spec with placement count:2 2024-03-21T12:27:54.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:54 smithi022 ceph-mon[24777]: from='mgr.14118 172.21.15.22:0/380713134' entity='mgr.smithi022.fektzs' 2024-03-21T12:27:55.201 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: Scheduled crash update... 2024-03-21T12:27:55.990 INFO:teuthology.orchestra.run.smithi022.stderr:Deploying prometheus service with default placement... 2024-03-21T12:27:56.258 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:56 smithi022 ceph-mon[24777]: from='client.14144 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "crash", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:27:56.259 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:56 smithi022 ceph-mon[24777]: Saving service crash spec with placement * 2024-03-21T12:27:56.259 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:56 smithi022 ceph-mon[24777]: from='mgr.14118 172.21.15.22:0/380713134' entity='mgr.smithi022.fektzs' 2024-03-21T12:27:56.259 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:56 smithi022 ceph-mon[24777]: from='mgr.14118 172.21.15.22:0/380713134' entity='mgr.smithi022.fektzs' 2024-03-21T12:27:56.259 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:56 smithi022 ceph-mon[24777]: from='mgr.14118 172.21.15.22:0/380713134' entity='mgr.smithi022.fektzs' 2024-03-21T12:27:56.662 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: Scheduled prometheus update... 2024-03-21T12:27:57.723 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:57 smithi022 ceph-mon[24777]: from='client.14146 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:27:57.724 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:57 smithi022 ceph-mon[24777]: Saving service prometheus spec with placement count:1 2024-03-21T12:27:57.724 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:57 smithi022 ceph-mon[24777]: from='mgr.14118 172.21.15.22:0/380713134' entity='mgr.smithi022.fektzs' 2024-03-21T12:27:57.724 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:57 smithi022 ceph-mon[24777]: from='mgr.14118 172.21.15.22:0/380713134' entity='mgr.smithi022.fektzs' 2024-03-21T12:27:57.825 INFO:teuthology.orchestra.run.smithi022.stderr:Deploying grafana service with default placement... 2024-03-21T12:27:58.476 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: Scheduled grafana update... 2024-03-21T12:27:59.014 INFO:teuthology.orchestra.run.smithi022.stderr:Deploying node-exporter service with default placement... 2024-03-21T12:27:59.439 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:59 smithi022 ceph-mon[24777]: from='mgr.14118 172.21.15.22:0/380713134' entity='mgr.smithi022.fektzs' 2024-03-21T12:27:59.439 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:59 smithi022 ceph-mon[24777]: from='client.14148 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:27:59.439 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:59 smithi022 ceph-mon[24777]: Saving service grafana spec with placement count:1 2024-03-21T12:27:59.440 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:27:59 smithi022 ceph-mon[24777]: from='mgr.14118 172.21.15.22:0/380713134' entity='mgr.smithi022.fektzs' 2024-03-21T12:27:59.663 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: Scheduled node-exporter update... 2024-03-21T12:28:00.222 INFO:teuthology.orchestra.run.smithi022.stderr:Deploying alertmanager service with default placement... 2024-03-21T12:28:00.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:00 smithi022 ceph-mon[24777]: from='client.14150 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:28:00.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:00 smithi022 ceph-mon[24777]: Saving service node-exporter spec with placement * 2024-03-21T12:28:00.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:00 smithi022 ceph-mon[24777]: from='mgr.14118 172.21.15.22:0/380713134' entity='mgr.smithi022.fektzs' 2024-03-21T12:28:01.309 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: Scheduled alertmanager update... 2024-03-21T12:28:02.325 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:02 smithi022 ceph-mon[24777]: from='client.14152 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:28:02.326 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:02 smithi022 ceph-mon[24777]: Saving service alertmanager spec with placement count:1 2024-03-21T12:28:02.326 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:02 smithi022 ceph-mon[24777]: from='mgr.14118 172.21.15.22:0/380713134' entity='mgr.smithi022.fektzs' 2024-03-21T12:28:03.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:03 smithi022 ceph-mon[24777]: from='client.? 172.21.15.22:0/1347548491' entity='client.admin' 2024-03-21T12:28:03.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:03 smithi022 ceph-mon[24777]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-21T12:28:04.579 INFO:teuthology.orchestra.run.smithi022.stderr:Enabling the dashboard module... 2024-03-21T12:28:05.054 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:04 smithi022 ceph-mon[24777]: from='client.? 172.21.15.22:0/2633678077' entity='client.admin' 2024-03-21T12:28:06.300 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:05 smithi022 ceph-mon[24777]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-21T12:28:06.300 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:05 smithi022 ceph-mon[24777]: from='client.? 172.21.15.22:0/3491224306' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-03-21T12:28:07.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:06 smithi022 ceph-mon[24777]: from='client.? 172.21.15.22:0/3491224306' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-03-21T12:28:07.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:06 smithi022 ceph-mon[24777]: mgrmap e9: smithi022.fektzs(active, since 23s) 2024-03-21T12:28:07.287 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: { 2024-03-21T12:28:07.287 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "epoch": 9, 2024-03-21T12:28:07.287 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "available": true, 2024-03-21T12:28:07.287 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "active_name": "smithi022.fektzs", 2024-03-21T12:28:07.287 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "num_standby": 0 2024-03-21T12:28:07.287 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: } 2024-03-21T12:28:07.926 INFO:teuthology.orchestra.run.smithi022.stderr:Waiting for the mgr to restart... 2024-03-21T12:28:07.926 INFO:teuthology.orchestra.run.smithi022.stderr:Waiting for mgr epoch 9... 2024-03-21T12:28:08.061 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:07 smithi022 ceph-mon[24777]: from='client.? 172.21.15.22:0/3385400500' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-03-21T12:28:13.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:12 smithi022 ceph-mon[24777]: Active manager daemon smithi022.fektzs restarted 2024-03-21T12:28:13.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:12 smithi022 ceph-mon[24777]: Activating manager daemon smithi022.fektzs 2024-03-21T12:28:13.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:12 smithi022 ceph-mon[24777]: osdmap e3: 0 total, 0 up, 0 in 2024-03-21T12:28:13.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:12 smithi022 ceph-mon[24777]: mgrmap e10: smithi022.fektzs(active, starting, since 0.802832s) 2024-03-21T12:28:13.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:12 smithi022 ceph-mon[24777]: from='mgr.14160 172.21.15.22:0/3694986284' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mon metadata", "id": "smithi022"}]: dispatch 2024-03-21T12:28:13.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:12 smithi022 ceph-mon[24777]: from='mgr.14160 172.21.15.22:0/3694986284' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mgr metadata", "who": "smithi022.fektzs", "id": "smithi022.fektzs"}]: dispatch 2024-03-21T12:28:13.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:12 smithi022 ceph-mon[24777]: from='mgr.14160 172.21.15.22:0/3694986284' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-21T12:28:13.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:12 smithi022 ceph-mon[24777]: from='mgr.14160 172.21.15.22:0/3694986284' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-21T12:28:13.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:12 smithi022 ceph-mon[24777]: from='mgr.14160 172.21.15.22:0/3694986284' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-21T12:28:13.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:12 smithi022 ceph-mon[24777]: Manager daemon smithi022.fektzs is now available 2024-03-21T12:28:13.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:12 smithi022 ceph-mon[24777]: from='mgr.14160 172.21.15.22:0/3694986284' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:28:13.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:12 smithi022 ceph-mon[24777]: from='mgr.14160 172.21.15.22:0/3694986284' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:28:13.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:12 smithi022 ceph-mon[24777]: from='mgr.14160 172.21.15.22:0/3694986284' entity='mgr.smithi022.fektzs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi022.fektzs/mirror_snapshot_schedule"}]: dispatch 2024-03-21T12:28:13.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:12 smithi022 ceph-mon[24777]: from='mgr.14160 172.21.15.22:0/3694986284' entity='mgr.smithi022.fektzs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi022.fektzs/trash_purge_schedule"}]: dispatch 2024-03-21T12:28:13.773 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: { 2024-03-21T12:28:13.773 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "mgrmap_epoch": 11, 2024-03-21T12:28:13.773 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "initialized": true 2024-03-21T12:28:13.774 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: } 2024-03-21T12:28:14.461 INFO:teuthology.orchestra.run.smithi022.stderr:mgr epoch 9 is available 2024-03-21T12:28:14.461 INFO:teuthology.orchestra.run.smithi022.stderr:Generating a dashboard self-signed certificate... 2024-03-21T12:28:15.082 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:14 smithi022 ceph-mon[24777]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-03-21T12:28:15.082 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:14 smithi022 ceph-mon[24777]: mgrmap e11: smithi022.fektzs(active, since 1.80501s) 2024-03-21T12:28:15.082 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:14 smithi022 ceph-mon[24777]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-03-21T12:28:15.082 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:14 smithi022 ceph-mon[24777]: from='mgr.14160 172.21.15.22:0/3694986284' entity='mgr.smithi022.fektzs' 2024-03-21T12:28:15.082 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:14 smithi022 ceph-mon[24777]: from='mgr.14160 172.21.15.22:0/3694986284' entity='mgr.smithi022.fektzs' 2024-03-21T12:28:15.198 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: Self-signed certificate created 2024-03-21T12:28:15.826 INFO:teuthology.orchestra.run.smithi022.stderr:Creating initial admin user... 2024-03-21T12:28:16.061 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:15 smithi022 ceph-mon[24777]: [21/Mar/2024:12:28:14] ENGINE Bus STARTING 2024-03-21T12:28:16.062 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:15 smithi022 ceph-mon[24777]: [21/Mar/2024:12:28:14] ENGINE Serving on https://172.21.15.22:7150 2024-03-21T12:28:16.062 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:15 smithi022 ceph-mon[24777]: [21/Mar/2024:12:28:14] ENGINE Bus STARTED 2024-03-21T12:28:16.062 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:15 smithi022 ceph-mon[24777]: mgrmap e12: smithi022.fektzs(active, since 2s) 2024-03-21T12:28:16.062 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:15 smithi022 ceph-mon[24777]: from='mgr.14160 172.21.15.22:0/3694986284' entity='mgr.smithi022.fektzs' 2024-03-21T12:28:16.062 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:15 smithi022 ceph-mon[24777]: from='mgr.14160 172.21.15.22:0/3694986284' entity='mgr.smithi022.fektzs' 2024-03-21T12:28:16.062 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:15 smithi022 ceph-mon[24777]: from='mgr.14160 172.21.15.22:0/3694986284' entity='mgr.smithi022.fektzs' 2024-03-21T12:28:16.732 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: {"username": "admin", "password": "$2b$12$TxGaCqbE7Iw6a03Ak/nGFeNpySxHc76wgQgkZdKy/o4WPv8mrQUFK", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1711024096, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-03-21T12:28:16.973 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:16 smithi022 ceph-mon[24777]: from='client.14172 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:28:16.973 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:16 smithi022 ceph-mon[24777]: from='mgr.14160 172.21.15.22:0/3694986284' entity='mgr.smithi022.fektzs' 2024-03-21T12:28:17.596 INFO:teuthology.orchestra.run.smithi022.stderr:Fetching dashboard port number... 2024-03-21T12:28:17.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:17 smithi022 ceph-mon[24777]: from='client.14174 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:28:18.861 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: 8443 2024-03-21T12:28:19.549 INFO:teuthology.orchestra.run.smithi022.stderr:systemctl: disabled 2024-03-21T12:28:19.557 INFO:teuthology.orchestra.run.smithi022.stderr:systemctl: inactive 2024-03-21T12:28:19.557 INFO:teuthology.orchestra.run.smithi022.stderr:firewalld.service is not enabled 2024-03-21T12:28:19.558 INFO:teuthology.orchestra.run.smithi022.stderr:Not possible to open ports <[8443]>. firewalld.service is not available 2024-03-21T12:28:19.560 INFO:teuthology.orchestra.run.smithi022.stderr:Ceph Dashboard is now available at: 2024-03-21T12:28:19.560 INFO:teuthology.orchestra.run.smithi022.stderr: 2024-03-21T12:28:19.560 INFO:teuthology.orchestra.run.smithi022.stderr: URL: https://smithi022.front.sepia.ceph.com:8443/ 2024-03-21T12:28:19.560 INFO:teuthology.orchestra.run.smithi022.stderr: User: admin 2024-03-21T12:28:19.560 INFO:teuthology.orchestra.run.smithi022.stderr: Password: ror355euo1 2024-03-21T12:28:19.561 INFO:teuthology.orchestra.run.smithi022.stderr: 2024-03-21T12:28:19.561 INFO:teuthology.orchestra.run.smithi022.stderr:Enabling autotune for osd_memory_target 2024-03-21T12:28:19.802 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:19 smithi022 ceph-mon[24777]: mgrmap e13: smithi022.fektzs(active, since 6s) 2024-03-21T12:28:19.802 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:19 smithi022 ceph-mon[24777]: from='client.? 172.21.15.22:0/3590374113' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-03-21T12:28:21.656 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: set mgr/dashboard/cluster/status 2024-03-21T12:28:22.270 INFO:teuthology.orchestra.run.smithi022.stderr:You can access the Ceph CLI with: 2024-03-21T12:28:22.270 INFO:teuthology.orchestra.run.smithi022.stderr: 2024-03-21T12:28:22.270 INFO:teuthology.orchestra.run.smithi022.stderr: sudo /home/ubuntu/cephtest/cephadm shell --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-03-21T12:28:22.270 INFO:teuthology.orchestra.run.smithi022.stderr: 2024-03-21T12:28:22.270 INFO:teuthology.orchestra.run.smithi022.stderr:Please consider enabling telemetry to help improve Ceph: 2024-03-21T12:28:22.270 INFO:teuthology.orchestra.run.smithi022.stderr: 2024-03-21T12:28:22.270 INFO:teuthology.orchestra.run.smithi022.stderr: ceph telemetry on 2024-03-21T12:28:22.270 INFO:teuthology.orchestra.run.smithi022.stderr: 2024-03-21T12:28:22.271 INFO:teuthology.orchestra.run.smithi022.stderr:For more information see: 2024-03-21T12:28:22.271 INFO:teuthology.orchestra.run.smithi022.stderr: 2024-03-21T12:28:22.271 INFO:teuthology.orchestra.run.smithi022.stderr: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-03-21T12:28:22.271 INFO:teuthology.orchestra.run.smithi022.stderr: 2024-03-21T12:28:22.271 INFO:teuthology.orchestra.run.smithi022.stderr:Bootstrap complete. 2024-03-21T12:28:22.310 INFO:tasks.cephadm:Fetching config... 2024-03-21T12:28:22.310 DEBUG:teuthology.orchestra.run.smithi022:> set -ex 2024-03-21T12:28:22.310 DEBUG:teuthology.orchestra.run.smithi022:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-03-21T12:28:22.326 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-03-21T12:28:22.326 DEBUG:teuthology.orchestra.run.smithi022:> set -ex 2024-03-21T12:28:22.326 DEBUG:teuthology.orchestra.run.smithi022:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-03-21T12:28:22.381 INFO:tasks.cephadm:Fetching mon keyring... 2024-03-21T12:28:22.381 DEBUG:teuthology.orchestra.run.smithi022:> set -ex 2024-03-21T12:28:22.382 DEBUG:teuthology.orchestra.run.smithi022:> sudo dd if=/var/lib/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/mon.smithi022/keyring of=/dev/stdout 2024-03-21T12:28:22.448 INFO:tasks.cephadm:Fetching pub ssh key... 2024-03-21T12:28:22.448 DEBUG:teuthology.orchestra.run.smithi022:> set -ex 2024-03-21T12:28:22.448 DEBUG:teuthology.orchestra.run.smithi022:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-03-21T12:28:22.503 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-03-21T12:28:22.503 DEBUG:teuthology.orchestra.run.smithi022:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDLbGnL+TM5FZ0aSz9hpkXB5DREeXTvOMWfa1snuqJwRNjvTwRSricaUWlot6BZHLt9McNb6BXfNpjTt/BB+XzZ07W6X6NS/XmaNL33eEBvrjWNu6WCmUtgKCIHdY22hZQiOMSmNlKchvKQY0EA5resbSft8gKocfBNjs5Y53wAW9cYHhTrce4UgvV4U5rz80Xjd151VE2m69eA6hNtRSZ9tEDcRGeQZPEiyGDvY/F1nxsSdQGCG+QbhhKNpBHrRpiXWOL9z4KVmV5AHmnMQ4ZgBg82jeDMyHCwsIbW2lbsmL7jnB/ew+2Ymw4ehLkwWxknh9U0fCES70wt56861k1SPSQMBeOdz5JxhGzIIb/E1Zbj4lx7ofSJe3YTXjm88Sbdfdp9b5s/XBCfrRgrssHPZ78V6JhwmQ8a3SPGkFYD/TnJmYvrJkmE6MIv15cHyy3VKc27Le2bswrCI1KoP8TKMZFh++7BL6QYrZgxP3X1TV3/F+zfP4jOOyF48hjx1i0= ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-03-21T12:28:22.582 INFO:teuthology.orchestra.run.smithi022.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDLbGnL+TM5FZ0aSz9hpkXB5DREeXTvOMWfa1snuqJwRNjvTwRSricaUWlot6BZHLt9McNb6BXfNpjTt/BB+XzZ07W6X6NS/XmaNL33eEBvrjWNu6WCmUtgKCIHdY22hZQiOMSmNlKchvKQY0EA5resbSft8gKocfBNjs5Y53wAW9cYHhTrce4UgvV4U5rz80Xjd151VE2m69eA6hNtRSZ9tEDcRGeQZPEiyGDvY/F1nxsSdQGCG+QbhhKNpBHrRpiXWOL9z4KVmV5AHmnMQ4ZgBg82jeDMyHCwsIbW2lbsmL7jnB/ew+2Ymw4ehLkwWxknh9U0fCES70wt56861k1SPSQMBeOdz5JxhGzIIb/E1Zbj4lx7ofSJe3YTXjm88Sbdfdp9b5s/XBCfrRgrssHPZ78V6JhwmQ8a3SPGkFYD/TnJmYvrJkmE6MIv15cHyy3VKc27Le2bswrCI1KoP8TKMZFh++7BL6QYrZgxP3X1TV3/F+zfP4jOOyF48hjx1i0= ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715 2024-03-21T12:28:22.604 DEBUG:teuthology.orchestra.run.smithi189:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDLbGnL+TM5FZ0aSz9hpkXB5DREeXTvOMWfa1snuqJwRNjvTwRSricaUWlot6BZHLt9McNb6BXfNpjTt/BB+XzZ07W6X6NS/XmaNL33eEBvrjWNu6WCmUtgKCIHdY22hZQiOMSmNlKchvKQY0EA5resbSft8gKocfBNjs5Y53wAW9cYHhTrce4UgvV4U5rz80Xjd151VE2m69eA6hNtRSZ9tEDcRGeQZPEiyGDvY/F1nxsSdQGCG+QbhhKNpBHrRpiXWOL9z4KVmV5AHmnMQ4ZgBg82jeDMyHCwsIbW2lbsmL7jnB/ew+2Ymw4ehLkwWxknh9U0fCES70wt56861k1SPSQMBeOdz5JxhGzIIb/E1Zbj4lx7ofSJe3YTXjm88Sbdfdp9b5s/XBCfrRgrssHPZ78V6JhwmQ8a3SPGkFYD/TnJmYvrJkmE6MIv15cHyy3VKc27Le2bswrCI1KoP8TKMZFh++7BL6QYrZgxP3X1TV3/F+zfP4jOOyF48hjx1i0= ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-03-21T12:28:22.652 INFO:teuthology.orchestra.run.smithi189.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDLbGnL+TM5FZ0aSz9hpkXB5DREeXTvOMWfa1snuqJwRNjvTwRSricaUWlot6BZHLt9McNb6BXfNpjTt/BB+XzZ07W6X6NS/XmaNL33eEBvrjWNu6WCmUtgKCIHdY22hZQiOMSmNlKchvKQY0EA5resbSft8gKocfBNjs5Y53wAW9cYHhTrce4UgvV4U5rz80Xjd151VE2m69eA6hNtRSZ9tEDcRGeQZPEiyGDvY/F1nxsSdQGCG+QbhhKNpBHrRpiXWOL9z4KVmV5AHmnMQ4ZgBg82jeDMyHCwsIbW2lbsmL7jnB/ew+2Ymw4ehLkwWxknh9U0fCES70wt56861k1SPSQMBeOdz5JxhGzIIb/E1Zbj4lx7ofSJe3YTXjm88Sbdfdp9b5s/XBCfrRgrssHPZ78V6JhwmQ8a3SPGkFYD/TnJmYvrJkmE6MIv15cHyy3VKc27Le2bswrCI1KoP8TKMZFh++7BL6QYrZgxP3X1TV3/F+zfP4jOOyF48hjx1i0= ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715 2024-03-21T12:28:22.667 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-03-21T12:28:22.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:22 smithi022 ceph-mon[24777]: from='client.? 172.21.15.22:0/20532979' entity='client.admin' 2024-03-21T12:28:26.951 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:26 smithi022 ceph-mon[24777]: from='client.? 172.21.15.22:0/268210841' entity='client.admin' 2024-03-21T12:28:26.952 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:26 smithi022 ceph-mon[24777]: from='mgr.14160 172.21.15.22:0/3694986284' entity='mgr.smithi022.fektzs' 2024-03-21T12:28:26.952 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:26 smithi022 ceph-mon[24777]: from='mgr.14160 172.21.15.22:0/3694986284' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "who": "osd/host:smithi022", "name": "osd_memory_target"}]: dispatch 2024-03-21T12:28:26.952 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:26 smithi022 ceph-mon[24777]: from='mgr.14160 172.21.15.22:0/3694986284' entity='mgr.smithi022.fektzs' 2024-03-21T12:28:26.974 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-03-21T12:28:26.974 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-03-21T12:28:27.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:27 smithi022 ceph-mon[24777]: Deploying daemon alertmanager.smithi022 on smithi022 2024-03-21T12:28:30.603 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi189 2024-03-21T12:28:30.603 DEBUG:teuthology.orchestra.run.smithi189:> set -ex 2024-03-21T12:28:30.603 DEBUG:teuthology.orchestra.run.smithi189:> dd of=/etc/ceph/ceph.conf 2024-03-21T12:28:30.622 DEBUG:teuthology.orchestra.run.smithi189:> set -ex 2024-03-21T12:28:30.623 DEBUG:teuthology.orchestra.run.smithi189:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-03-21T12:28:30.681 INFO:tasks.cephadm:Adding host smithi189 to orchestrator... 2024-03-21T12:28:30.681 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -- ceph orch host add smithi189 2024-03-21T12:28:30.706 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:30 smithi022 ceph-mon[24777]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:28:30.706 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:30 smithi022 ceph-mon[24777]: from='mgr.14160 172.21.15.22:0/3694986284' entity='mgr.smithi022.fektzs' 2024-03-21T12:28:33.920 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:33 smithi022 ceph-mon[24777]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-21T12:28:36.021 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:35 smithi022 ceph-mon[24777]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-21T12:28:36.883 INFO:teuthology.orchestra.run.smithi022.stdout:Added host 'smithi189' with addr '172.21.15.189' 2024-03-21T12:28:36.944 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:36 smithi022 ceph-mon[24777]: from='client.14186 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi189", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:28:36.944 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:36 smithi022 ceph-mon[24777]: from='mgr.14160 172.21.15.22:0/3694986284' entity='mgr.smithi022.fektzs' 2024-03-21T12:28:36.944 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:36 smithi022 ceph-mon[24777]: from='mgr.14160 172.21.15.22:0/3694986284' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi022", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-03-21T12:28:36.944 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:36 smithi022 ceph-mon[24777]: from='mgr.14160 172.21.15.22:0/3694986284' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi022", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-03-21T12:28:36.945 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:36 smithi022 ceph-mon[24777]: from='mgr.14160 172.21.15.22:0/3694986284' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:28:37.844 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -- ceph orch host ls --format=json 2024-03-21T12:28:38.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:37 smithi022 ceph-mon[24777]: Deploying daemon crash.smithi022 on smithi022 2024-03-21T12:28:38.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:37 smithi022 ceph-mon[24777]: Deploying cephadm binary to smithi189 2024-03-21T12:28:38.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:37 smithi022 ceph-mon[24777]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-21T12:28:38.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:37 smithi022 ceph-mon[24777]: from='mgr.14160 172.21.15.22:0/3694986284' entity='mgr.smithi022.fektzs' 2024-03-21T12:28:38.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:37 smithi022 ceph-mon[24777]: Added host smithi189 2024-03-21T12:28:38.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:37 smithi022 ceph-mon[24777]: from='mgr.14160 172.21.15.22:0/3694986284' entity='mgr.smithi022.fektzs' 2024-03-21T12:28:40.051 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:39 smithi022 ceph-mon[24777]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-21T12:28:40.103 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-03-21T12:28:40.103 INFO:teuthology.orchestra.run.smithi022.stdout:[{"addr": "172.21.15.22", "hostname": "smithi022", "labels": [], "status": ""}, {"addr": "172.21.15.189", "hostname": "smithi189", "labels": [], "status": ""}] 2024-03-21T12:28:40.722 INFO:tasks.cephadm:Setting crush tunables to default 2024-03-21T12:28:40.722 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -- ceph osd crush tunables default 2024-03-21T12:28:42.243 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:41 smithi022 ceph-mon[24777]: from='client.14188 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-21T12:28:42.243 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:41 smithi022 ceph-mon[24777]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-21T12:28:42.243 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:41 smithi022 ceph-mon[24777]: from='mgr.14160 172.21.15.22:0/3694986284' entity='mgr.smithi022.fektzs' 2024-03-21T12:28:42.243 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:41 smithi022 ceph-mon[24777]: from='mgr.14160 172.21.15.22:0/3694986284' entity='mgr.smithi022.fektzs' 2024-03-21T12:28:42.243 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:41 smithi022 ceph-mon[24777]: from='mgr.14160 172.21.15.22:0/3694986284' entity='mgr.smithi022.fektzs' 2024-03-21T12:28:42.243 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:41 smithi022 ceph-mon[24777]: from='mgr.14160 172.21.15.22:0/3694986284' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-03-21T12:28:42.243 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:41 smithi022 ceph-mon[24777]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-03-21T12:28:42.243 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:41 smithi022 ceph-mon[24777]: from='mgr.14160 172.21.15.22:0/3694986284' entity='mgr.smithi022.fektzs' 2024-03-21T12:28:42.243 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:41 smithi022 ceph-mon[24777]: Deploying daemon grafana.smithi022 on smithi022 2024-03-21T12:28:42.956 INFO:teuthology.orchestra.run.smithi022.stderr:adjusted tunables profile to default 2024-03-21T12:28:43.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:42 smithi022 ceph-mon[24777]: from='client.? 172.21.15.22:0/328289064' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-03-21T12:28:43.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:42 smithi022 ceph-mon[24777]: from='mgr.14160 172.21.15.22:0/3694986284' entity='mgr.smithi022.fektzs' 2024-03-21T12:28:43.677 INFO:tasks.cephadm:Adding mon.smithi022 on smithi022 2024-03-21T12:28:43.677 INFO:tasks.cephadm:Adding mon.smithi189 on smithi189 2024-03-21T12:28:43.677 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -- ceph orch apply mon '2;smithi022:172.21.15.22=smithi022;smithi189:172.21.15.189=smithi189' 2024-03-21T12:28:44.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:43 smithi022 ceph-mon[24777]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-21T12:28:44.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:43 smithi022 ceph-mon[24777]: from='client.? 172.21.15.22:0/328289064' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-03-21T12:28:44.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:43 smithi022 ceph-mon[24777]: osdmap e4: 0 total, 0 up, 0 in 2024-03-21T12:28:45.413 INFO:teuthology.orchestra.run.smithi189.stdout:Scheduled mon update... 2024-03-21T12:28:45.849 DEBUG:teuthology.orchestra.run.smithi189:mon.smithi189> sudo journalctl -f -n 0 -u ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715@mon.smithi189.service 2024-03-21T12:28:45.852 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-03-21T12:28:45.852 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -- ceph mon dump -f json 2024-03-21T12:28:46.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:45 smithi022 ceph-mon[24777]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-21T12:28:46.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:45 smithi022 ceph-mon[24777]: from='mgr.14160 172.21.15.22:0/3694986284' entity='mgr.smithi022.fektzs' 2024-03-21T12:28:47.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:46 smithi022 ceph-mon[24777]: from='client.14192 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "2;smithi022:172.21.15.22=smithi022;smithi189:172.21.15.189=smithi189", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:28:47.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:46 smithi022 ceph-mon[24777]: Saving service mon spec with placement smithi022:172.21.15.22=smithi022;smithi189:172.21.15.189=smithi189;count:2 2024-03-21T12:28:47.889 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-03-21T12:28:47.889 INFO:teuthology.orchestra.run.smithi189.stdout:{"epoch":1,"fsid":"3fc7f0f2-e77e-11ee-95cd-87774f69a715","modified":"2024-03-21T12:27:06.044181Z","created":"2024-03-21T12:27:06.044181Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi022","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:3300","nonce":0},{"type":"v1","addr":"172.21.15.22:6789","nonce":0}]},"addr":"172.21.15.22:6789/0","public_addr":"172.21.15.22:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-03-21T12:28:47.891 INFO:teuthology.orchestra.run.smithi189.stderr:dumped monmap epoch 1 2024-03-21T12:28:48.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:47 smithi022 ceph-mon[24777]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-21T12:28:48.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:47 smithi022 ceph-mon[24777]: from='client.? 172.21.15.189:0/1591455882' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-21T12:28:49.386 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-03-21T12:28:49.386 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -- ceph mon dump -f json 2024-03-21T12:28:50.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:49 smithi022 ceph-mon[24777]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-21T12:28:51.156 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-03-21T12:28:51.157 INFO:teuthology.orchestra.run.smithi189.stdout:{"epoch":1,"fsid":"3fc7f0f2-e77e-11ee-95cd-87774f69a715","modified":"2024-03-21T12:27:06.044181Z","created":"2024-03-21T12:27:06.044181Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi022","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:3300","nonce":0},{"type":"v1","addr":"172.21.15.22:6789","nonce":0}]},"addr":"172.21.15.22:6789/0","public_addr":"172.21.15.22:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-03-21T12:28:51.158 INFO:teuthology.orchestra.run.smithi189.stderr:dumped monmap epoch 1 2024-03-21T12:28:52.279 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:51 smithi022 ceph-mon[24777]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-21T12:28:52.279 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:51 smithi022 ceph-mon[24777]: from='client.? 172.21.15.189:0/1963296779' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-21T12:28:52.722 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-03-21T12:28:52.722 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -- ceph mon dump -f json 2024-03-21T12:28:54.263 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:53 smithi022 ceph-mon[24777]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-21T12:28:54.578 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-03-21T12:28:54.579 INFO:teuthology.orchestra.run.smithi189.stdout:{"epoch":1,"fsid":"3fc7f0f2-e77e-11ee-95cd-87774f69a715","modified":"2024-03-21T12:27:06.044181Z","created":"2024-03-21T12:27:06.044181Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi022","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:3300","nonce":0},{"type":"v1","addr":"172.21.15.22:6789","nonce":0}]},"addr":"172.21.15.22:6789/0","public_addr":"172.21.15.22:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-03-21T12:28:54.580 INFO:teuthology.orchestra.run.smithi189.stderr:dumped monmap epoch 1 2024-03-21T12:28:55.160 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:54 smithi022 ceph-mon[24777]: from='client.? 172.21.15.189:0/639131208' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-21T12:28:56.075 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-03-21T12:28:56.075 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -- ceph mon dump -f json 2024-03-21T12:28:56.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:55 smithi022 ceph-mon[24777]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-21T12:28:56.345 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:55 smithi022 ceph-mon[24777]: from='mgr.14160 172.21.15.22:0/3694986284' entity='mgr.smithi022.fektzs' 2024-03-21T12:28:56.345 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:55 smithi022 ceph-mon[24777]: Deploying daemon node-exporter.smithi022 on smithi022 2024-03-21T12:28:57.930 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-03-21T12:28:57.930 INFO:teuthology.orchestra.run.smithi189.stdout:{"epoch":1,"fsid":"3fc7f0f2-e77e-11ee-95cd-87774f69a715","modified":"2024-03-21T12:27:06.044181Z","created":"2024-03-21T12:27:06.044181Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi022","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:3300","nonce":0},{"type":"v1","addr":"172.21.15.22:6789","nonce":0}]},"addr":"172.21.15.22:6789/0","public_addr":"172.21.15.22:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-03-21T12:28:57.932 INFO:teuthology.orchestra.run.smithi189.stderr:dumped monmap epoch 1 2024-03-21T12:28:58.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:57 smithi022 ceph-mon[24777]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-21T12:28:59.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:58 smithi022 ceph-mon[24777]: from='mgr.14160 172.21.15.22:0/3694986284' entity='mgr.smithi022.fektzs' 2024-03-21T12:28:59.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:58 smithi022 ceph-mon[24777]: from='client.? 172.21.15.189:0/234998129' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-21T12:28:59.514 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-03-21T12:28:59.514 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -- ceph mon dump -f json 2024-03-21T12:29:00.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:59 smithi022 ceph-mon[24777]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-21T12:29:00.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:59 smithi022 ceph-mon[24777]: from='mgr.14160 172.21.15.22:0/3694986284' entity='mgr.smithi022.fektzs' 2024-03-21T12:29:00.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:28:59 smithi022 ceph-mon[24777]: from='mgr.14160 172.21.15.22:0/3694986284' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-03-21T12:29:01.207 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-03-21T12:29:01.207 INFO:teuthology.orchestra.run.smithi189.stdout:{"epoch":1,"fsid":"3fc7f0f2-e77e-11ee-95cd-87774f69a715","modified":"2024-03-21T12:27:06.044181Z","created":"2024-03-21T12:27:06.044181Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi022","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:3300","nonce":0},{"type":"v1","addr":"172.21.15.22:6789","nonce":0}]},"addr":"172.21.15.22:6789/0","public_addr":"172.21.15.22:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-03-21T12:29:01.209 INFO:teuthology.orchestra.run.smithi189.stderr:dumped monmap epoch 1 2024-03-21T12:29:02.065 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:01 smithi022 ceph-mon[24777]: from='mgr.14160 172.21.15.22:0/3694986284' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-03-21T12:29:02.065 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:01 smithi022 ceph-mon[24777]: mgrmap e14: smithi022.fektzs(active, since 48s) 2024-03-21T12:29:02.065 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:01 smithi022 ceph-mon[24777]: from='client.? 172.21.15.189:0/284846101' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-21T12:29:02.830 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-03-21T12:29:02.830 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -- ceph mon dump -f json 2024-03-21T12:29:04.562 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-03-21T12:29:04.562 INFO:teuthology.orchestra.run.smithi189.stdout:{"epoch":1,"fsid":"3fc7f0f2-e77e-11ee-95cd-87774f69a715","modified":"2024-03-21T12:27:06.044181Z","created":"2024-03-21T12:27:06.044181Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi022","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:3300","nonce":0},{"type":"v1","addr":"172.21.15.22:6789","nonce":0}]},"addr":"172.21.15.22:6789/0","public_addr":"172.21.15.22:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-03-21T12:29:04.564 INFO:teuthology.orchestra.run.smithi189.stderr:dumped monmap epoch 1 2024-03-21T12:29:04.982 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:04 smithi022 ceph-mon[24777]: from='client.? 172.21.15.189:0/903569062' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-21T12:29:06.014 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-03-21T12:29:06.014 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -- ceph mon dump -f json 2024-03-21T12:29:08.038 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:07 smithi022 ceph-mon[24777]: Active manager daemon smithi022.fektzs restarted 2024-03-21T12:29:08.039 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:07 smithi022 ceph-mon[24777]: Activating manager daemon smithi022.fektzs 2024-03-21T12:29:08.039 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:07 smithi022 ceph-mon[24777]: osdmap e5: 0 total, 0 up, 0 in 2024-03-21T12:29:08.039 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:07 smithi022 ceph-mon[24777]: mgrmap e15: smithi022.fektzs(active, starting, since 0.0524966s) 2024-03-21T12:29:08.039 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:07 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mon metadata", "id": "smithi022"}]: dispatch 2024-03-21T12:29:08.039 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:07 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mgr metadata", "who": "smithi022.fektzs", "id": "smithi022.fektzs"}]: dispatch 2024-03-21T12:29:08.039 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:07 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-21T12:29:08.039 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:07 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-21T12:29:08.039 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:07 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-21T12:29:08.039 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:07 smithi022 ceph-mon[24777]: Manager daemon smithi022.fektzs is now available 2024-03-21T12:29:08.039 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:07 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:29:08.039 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:07 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:29:08.039 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:07 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:29:08.040 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:07 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:29:08.040 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:07 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:29:08.040 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:07 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi022.fektzs/mirror_snapshot_schedule"}]: dispatch 2024-03-21T12:29:08.040 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:07 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi022.fektzs/trash_purge_schedule"}]: dispatch 2024-03-21T12:29:08.040 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:07 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:29:08.054 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-03-21T12:29:08.055 INFO:teuthology.orchestra.run.smithi189.stdout:{"epoch":1,"fsid":"3fc7f0f2-e77e-11ee-95cd-87774f69a715","modified":"2024-03-21T12:27:06.044181Z","created":"2024-03-21T12:27:06.044181Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi022","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:3300","nonce":0},{"type":"v1","addr":"172.21.15.22:6789","nonce":0}]},"addr":"172.21.15.22:6789/0","public_addr":"172.21.15.22:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-03-21T12:29:08.057 INFO:teuthology.orchestra.run.smithi189.stderr:dumped monmap epoch 1 2024-03-21T12:29:09.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:08 smithi022 ceph-mon[24777]: mgrmap e16: smithi022.fektzs(active, since 1.05537s) 2024-03-21T12:29:09.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:08 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:29:09.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:08 smithi022 ceph-mon[24777]: from='client.? 172.21.15.189:0/4140984701' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-21T12:29:09.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:08 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:29:09.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:08 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:29:09.719 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-03-21T12:29:09.719 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -- ceph mon dump -f json 2024-03-21T12:29:10.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:09 smithi022 ceph-mon[24777]: [21/Mar/2024:12:29:08] ENGINE Bus STARTING 2024-03-21T12:29:10.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:09 smithi022 ceph-mon[24777]: [21/Mar/2024:12:29:08] ENGINE Serving on https://172.21.15.22:7150 2024-03-21T12:29:10.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:09 smithi022 ceph-mon[24777]: [21/Mar/2024:12:29:08] ENGINE Bus STARTED 2024-03-21T12:29:10.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:09 smithi022 ceph-mon[24777]: mgrmap e17: smithi022.fektzs(active, since 2s) 2024-03-21T12:29:11.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:11 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:29:11.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:11 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:29:11.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:11 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "who": "osd/host:smithi022", "name": "osd_memory_target"}]: dispatch 2024-03-21T12:29:12.148 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-03-21T12:29:12.148 INFO:teuthology.orchestra.run.smithi189.stdout:{"epoch":1,"fsid":"3fc7f0f2-e77e-11ee-95cd-87774f69a715","modified":"2024-03-21T12:27:06.044181Z","created":"2024-03-21T12:27:06.044181Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi022","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:3300","nonce":0},{"type":"v1","addr":"172.21.15.22:6789","nonce":0}]},"addr":"172.21.15.22:6789/0","public_addr":"172.21.15.22:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-03-21T12:29:12.150 INFO:teuthology.orchestra.run.smithi189.stderr:dumped monmap epoch 1 2024-03-21T12:29:13.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:12 smithi022 ceph-mon[24777]: Updating smithi022:/etc/ceph/ceph.conf 2024-03-21T12:29:13.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:12 smithi022 ceph-mon[24777]: Updating smithi022:/etc/ceph/ceph.client.admin.keyring 2024-03-21T12:29:13.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:12 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:29:13.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:12 smithi022 ceph-mon[24777]: from='client.? 172.21.15.189:0/2232235318' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-21T12:29:13.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:12 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:29:13.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:12 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:29:13.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:12 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "who": "osd/host:smithi189", "name": "osd_memory_target"}]: dispatch 2024-03-21T12:29:13.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:12 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:29:13.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:12 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi189", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-03-21T12:29:13.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:12 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi189", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-03-21T12:29:13.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:12 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:29:13.826 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-03-21T12:29:13.827 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -- ceph mon dump -f json 2024-03-21T12:29:14.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:13 smithi022 ceph-mon[24777]: Deploying daemon crash.smithi189 on smithi189 2024-03-21T12:29:16.433 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-03-21T12:29:16.434 INFO:teuthology.orchestra.run.smithi189.stdout:{"epoch":1,"fsid":"3fc7f0f2-e77e-11ee-95cd-87774f69a715","modified":"2024-03-21T12:27:06.044181Z","created":"2024-03-21T12:27:06.044181Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi022","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:3300","nonce":0},{"type":"v1","addr":"172.21.15.22:6789","nonce":0}]},"addr":"172.21.15.22:6789/0","public_addr":"172.21.15.22:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-03-21T12:29:16.435 INFO:teuthology.orchestra.run.smithi189.stderr:dumped monmap epoch 1 2024-03-21T12:29:17.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:16 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:29:17.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:16 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi189.cnxzvu", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-21T12:29:17.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:16 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.smithi189.cnxzvu", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-03-21T12:29:17.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:16 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-21T12:29:17.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:16 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:29:17.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:16 smithi022 ceph-mon[24777]: Deploying daemon mgr.smithi189.cnxzvu on smithi189 2024-03-21T12:29:17.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:16 smithi022 ceph-mon[24777]: from='client.? 172.21.15.189:0/804045220' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-21T12:29:17.974 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-03-21T12:29:17.975 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -- ceph mon dump -f json 2024-03-21T12:29:18.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:17 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:29:20.413 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-03-21T12:29:20.413 INFO:teuthology.orchestra.run.smithi189.stdout:{"epoch":1,"fsid":"3fc7f0f2-e77e-11ee-95cd-87774f69a715","modified":"2024-03-21T12:27:06.044181Z","created":"2024-03-21T12:27:06.044181Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi022","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:3300","nonce":0},{"type":"v1","addr":"172.21.15.22:6789","nonce":0}]},"addr":"172.21.15.22:6789/0","public_addr":"172.21.15.22:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-03-21T12:29:20.415 INFO:teuthology.orchestra.run.smithi189.stderr:dumped monmap epoch 1 2024-03-21T12:29:21.048 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:20 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:29:21.048 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:20 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-21T12:29:21.048 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:20 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:29:21.048 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:20 smithi022 ceph-mon[24777]: Deploying daemon mon.smithi189 on smithi189 2024-03-21T12:29:21.048 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:20 smithi022 ceph-mon[24777]: from='client.? 172.21.15.189:0/3880328977' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-21T12:29:22.254 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-03-21T12:29:22.254 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -- ceph mon dump -f json 2024-03-21T12:29:23.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:22 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:29:24.588 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-03-21T12:29:24.588 INFO:teuthology.orchestra.run.smithi189.stdout:{"epoch":1,"fsid":"3fc7f0f2-e77e-11ee-95cd-87774f69a715","modified":"2024-03-21T12:27:06.044181Z","created":"2024-03-21T12:27:06.044181Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi022","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:3300","nonce":0},{"type":"v1","addr":"172.21.15.22:6789","nonce":0}]},"addr":"172.21.15.22:6789/0","public_addr":"172.21.15.22:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-03-21T12:29:24.590 INFO:teuthology.orchestra.run.smithi189.stderr:dumped monmap epoch 1 2024-03-21T12:29:25.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:24 smithi022 ceph-mon[24777]: from='client.? 172.21.15.189:0/2838886291' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-21T12:29:26.603 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-03-21T12:29:26.604 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -- ceph mon dump -f json 2024-03-21T12:29:30.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:30 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mon metadata", "id": "smithi022"}]: dispatch 2024-03-21T12:29:30.345 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:30 smithi022 ceph-mon[24777]: mon.smithi022 calling monitor election 2024-03-21T12:29:30.345 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:30 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mon metadata", "id": "smithi189"}]: dispatch 2024-03-21T12:29:30.345 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:30 smithi022 ceph-mon[24777]: from='mgr.? 172.21.15.189:0/1594862793' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi189.cnxzvu/crt"}]: dispatch 2024-03-21T12:29:30.345 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:30 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mon metadata", "id": "smithi189"}]: dispatch 2024-03-21T12:29:30.345 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:30 smithi022 ceph-mon[24777]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-21T12:29:30.345 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:30 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mon metadata", "id": "smithi189"}]: dispatch 2024-03-21T12:29:30.345 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:30 smithi022 ceph-mon[24777]: mon.smithi189 calling monitor election 2024-03-21T12:29:30.345 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:30 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mon metadata", "id": "smithi189"}]: dispatch 2024-03-21T12:29:30.345 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:30 smithi022 ceph-mon[24777]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-21T12:29:30.345 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:30 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mon metadata", "id": "smithi189"}]: dispatch 2024-03-21T12:29:30.345 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:30 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mon metadata", "id": "smithi189"}]: dispatch 2024-03-21T12:29:30.345 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:30 smithi022 ceph-mon[24777]: mon.smithi022 is new leader, mons smithi022,smithi189 in quorum (ranks 0,1) 2024-03-21T12:29:30.346 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:30 smithi022 ceph-mon[24777]: monmap e2: 2 mons at {smithi022=[v2:172.21.15.22:3300/0,v1:172.21.15.22:6789/0],smithi189=[v2:172.21.15.189:3300/0,v1:172.21.15.189:6789/0]} 2024-03-21T12:29:30.346 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:30 smithi022 ceph-mon[24777]: fsmap 2024-03-21T12:29:30.346 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:30 smithi022 ceph-mon[24777]: osdmap e5: 0 total, 0 up, 0 in 2024-03-21T12:29:30.346 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:30 smithi022 ceph-mon[24777]: mgrmap e17: smithi022.fektzs(active, since 23s) 2024-03-21T12:29:30.346 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:30 smithi022 ceph-mon[24777]: Standby manager daemon smithi189.cnxzvu started 2024-03-21T12:29:30.346 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:30 smithi022 ceph-mon[24777]: from='mgr.? 172.21.15.189:0/1594862793' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-21T12:29:30.346 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:30 smithi022 ceph-mon[24777]: overall HEALTH_OK 2024-03-21T12:29:30.346 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:30 smithi022 ceph-mon[24777]: from='mgr.? 172.21.15.189:0/1594862793' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi189.cnxzvu/key"}]: dispatch 2024-03-21T12:29:30.346 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:30 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:29:30.346 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:30 smithi022 ceph-mon[24777]: from='mgr.? 172.21.15.189:0/1594862793' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-21T12:29:31.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:31 smithi022 ceph-mon[24777]: Deploying daemon node-exporter.smithi189 on smithi189 2024-03-21T12:29:31.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:31 smithi022 ceph-mon[24777]: mgrmap e18: smithi022.fektzs(active, since 23s), standbys: smithi189.cnxzvu 2024-03-21T12:29:31.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:31 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mgr metadata", "who": "smithi189.cnxzvu", "id": "smithi189.cnxzvu"}]: dispatch 2024-03-21T12:29:31.345 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:31 smithi022 ceph-mon[24777]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-21T12:29:31.345 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:31 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mon metadata", "id": "smithi189"}]: dispatch 2024-03-21T12:29:31.400 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-03-21T12:29:31.400 INFO:teuthology.orchestra.run.smithi189.stdout:{"epoch":2,"fsid":"3fc7f0f2-e77e-11ee-95cd-87774f69a715","modified":"2024-03-21T12:29:24.965263Z","created":"2024-03-21T12:27:06.044181Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi022","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:3300","nonce":0},{"type":"v1","addr":"172.21.15.22:6789","nonce":0}]},"addr":"172.21.15.22:6789/0","public_addr":"172.21.15.22:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"smithi189","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:3300","nonce":0},{"type":"v1","addr":"172.21.15.189:6789","nonce":0}]},"addr":"172.21.15.189:6789/0","public_addr":"172.21.15.189:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-03-21T12:29:31.401 INFO:teuthology.orchestra.run.smithi189.stderr:dumped monmap epoch 2 2024-03-21T12:29:32.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:32 smithi022 ceph-mon[24777]: from='client.? 172.21.15.189:0/1426176339' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-21T12:29:32.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:32 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:29:32.513 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:29:32 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:29:33.143 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:29:33 smithi189 ceph-mon[27080]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-21T12:29:33.165 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-03-21T12:29:33.165 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -- ceph config generate-minimal-conf 2024-03-21T12:29:33.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:33 smithi022 ceph-mon[24777]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-21T12:29:34.727 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:34 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:29:34.727 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:34 smithi022 ceph-mon[24777]: Deploying daemon prometheus.smithi022 on smithi022 2024-03-21T12:29:35.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:29:34 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:29:35.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:29:34 smithi189 ceph-mon[27080]: Deploying daemon prometheus.smithi022 on smithi022 2024-03-21T12:29:35.156 INFO:teuthology.orchestra.run.smithi022.stdout:# minimal ceph.conf for 3fc7f0f2-e77e-11ee-95cd-87774f69a715 2024-03-21T12:29:35.156 INFO:teuthology.orchestra.run.smithi022.stdout:[global] 2024-03-21T12:29:35.156 INFO:teuthology.orchestra.run.smithi022.stdout: fsid = 3fc7f0f2-e77e-11ee-95cd-87774f69a715 2024-03-21T12:29:35.156 INFO:teuthology.orchestra.run.smithi022.stdout: mon_host = [v2:172.21.15.22:3300/0,v1:172.21.15.22:6789/0] [v2:172.21.15.189:3300/0,v1:172.21.15.189:6789/0] 2024-03-21T12:29:35.834 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:35 smithi022 ceph-mon[24777]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-21T12:29:35.834 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:35 smithi022 ceph-mon[24777]: from='client.? 172.21.15.22:0/4203860948' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:29:35.854 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-03-21T12:29:35.854 DEBUG:teuthology.orchestra.run.smithi022:> set -ex 2024-03-21T12:29:35.854 DEBUG:teuthology.orchestra.run.smithi022:> sudo dd of=/etc/ceph/ceph.conf 2024-03-21T12:29:35.884 DEBUG:teuthology.orchestra.run.smithi022:> set -ex 2024-03-21T12:29:35.884 DEBUG:teuthology.orchestra.run.smithi022:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-03-21T12:29:35.953 DEBUG:teuthology.orchestra.run.smithi189:> set -ex 2024-03-21T12:29:35.953 DEBUG:teuthology.orchestra.run.smithi189:> sudo dd of=/etc/ceph/ceph.conf 2024-03-21T12:29:35.983 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:29:35 smithi189 ceph-mon[27080]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-21T12:29:35.983 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:29:35 smithi189 ceph-mon[27080]: from='client.? 172.21.15.22:0/4203860948' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:29:35.989 DEBUG:teuthology.orchestra.run.smithi189:> set -ex 2024-03-21T12:29:35.989 DEBUG:teuthology.orchestra.run.smithi189:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-03-21T12:29:36.059 INFO:tasks.cephadm:Deploying OSDs... 2024-03-21T12:29:36.059 DEBUG:teuthology.orchestra.run.smithi022:> set -ex 2024-03-21T12:29:36.059 DEBUG:teuthology.orchestra.run.smithi022:> dd if=/scratch_devs of=/dev/stdout 2024-03-21T12:29:36.074 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-03-21T12:29:36.074 DEBUG:teuthology.orchestra.run.smithi022:> stat /dev/vg_nvme/lv_1 2024-03-21T12:29:36.130 INFO:teuthology.orchestra.run.smithi022.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-03-21T12:29:36.131 INFO:teuthology.orchestra.run.smithi022.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-21T12:29:36.131 INFO:teuthology.orchestra.run.smithi022.stdout:Device: 5h/5d Inode: 666 Links: 1 2024-03-21T12:29:36.131 INFO:teuthology.orchestra.run.smithi022.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-21T12:29:36.131 INFO:teuthology.orchestra.run.smithi022.stdout:Context: system_u:object_r:device_t:s0 2024-03-21T12:29:36.131 INFO:teuthology.orchestra.run.smithi022.stdout:Access: 2024-03-21 12:28:55.873504803 +0000 2024-03-21T12:29:36.131 INFO:teuthology.orchestra.run.smithi022.stdout:Modify: 2024-03-21 12:26:09.186980194 +0000 2024-03-21T12:29:36.131 INFO:teuthology.orchestra.run.smithi022.stdout:Change: 2024-03-21 12:26:09.186980194 +0000 2024-03-21T12:29:36.131 INFO:teuthology.orchestra.run.smithi022.stdout: Birth: 2024-03-21 12:26:09.186980194 +0000 2024-03-21T12:29:36.131 DEBUG:teuthology.orchestra.run.smithi022:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-03-21T12:29:36.199 INFO:teuthology.orchestra.run.smithi022.stderr:1+0 records in 2024-03-21T12:29:36.199 INFO:teuthology.orchestra.run.smithi022.stderr:1+0 records out 2024-03-21T12:29:36.199 INFO:teuthology.orchestra.run.smithi022.stderr:512 bytes copied, 9.2916e-05 s, 5.5 MB/s 2024-03-21T12:29:36.200 DEBUG:teuthology.orchestra.run.smithi022:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-03-21T12:29:36.256 DEBUG:teuthology.orchestra.run.smithi022:> stat /dev/vg_nvme/lv_2 2024-03-21T12:29:36.311 INFO:teuthology.orchestra.run.smithi022.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-03-21T12:29:36.312 INFO:teuthology.orchestra.run.smithi022.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-21T12:29:36.312 INFO:teuthology.orchestra.run.smithi022.stdout:Device: 5h/5d Inode: 687 Links: 1 2024-03-21T12:29:36.312 INFO:teuthology.orchestra.run.smithi022.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-21T12:29:36.312 INFO:teuthology.orchestra.run.smithi022.stdout:Context: system_u:object_r:device_t:s0 2024-03-21T12:29:36.312 INFO:teuthology.orchestra.run.smithi022.stdout:Access: 2024-03-21 12:28:55.874504790 +0000 2024-03-21T12:29:36.312 INFO:teuthology.orchestra.run.smithi022.stdout:Modify: 2024-03-21 12:26:09.520975786 +0000 2024-03-21T12:29:36.312 INFO:teuthology.orchestra.run.smithi022.stdout:Change: 2024-03-21 12:26:09.520975786 +0000 2024-03-21T12:29:36.312 INFO:teuthology.orchestra.run.smithi022.stdout: Birth: 2024-03-21 12:26:09.520975786 +0000 2024-03-21T12:29:36.312 DEBUG:teuthology.orchestra.run.smithi022:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-03-21T12:29:36.378 INFO:teuthology.orchestra.run.smithi022.stderr:1+0 records in 2024-03-21T12:29:36.378 INFO:teuthology.orchestra.run.smithi022.stderr:1+0 records out 2024-03-21T12:29:36.378 INFO:teuthology.orchestra.run.smithi022.stderr:512 bytes copied, 8.3958e-05 s, 6.1 MB/s 2024-03-21T12:29:36.379 DEBUG:teuthology.orchestra.run.smithi022:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-03-21T12:29:36.434 DEBUG:teuthology.orchestra.run.smithi022:> stat /dev/vg_nvme/lv_3 2024-03-21T12:29:36.490 INFO:teuthology.orchestra.run.smithi022.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-03-21T12:29:36.490 INFO:teuthology.orchestra.run.smithi022.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-21T12:29:36.490 INFO:teuthology.orchestra.run.smithi022.stdout:Device: 5h/5d Inode: 705 Links: 1 2024-03-21T12:29:36.490 INFO:teuthology.orchestra.run.smithi022.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-21T12:29:36.490 INFO:teuthology.orchestra.run.smithi022.stdout:Context: system_u:object_r:device_t:s0 2024-03-21T12:29:36.490 INFO:teuthology.orchestra.run.smithi022.stdout:Access: 2024-03-21 12:28:55.874504790 +0000 2024-03-21T12:29:36.490 INFO:teuthology.orchestra.run.smithi022.stdout:Modify: 2024-03-21 12:26:09.871971154 +0000 2024-03-21T12:29:36.490 INFO:teuthology.orchestra.run.smithi022.stdout:Change: 2024-03-21 12:26:09.871971154 +0000 2024-03-21T12:29:36.490 INFO:teuthology.orchestra.run.smithi022.stdout: Birth: 2024-03-21 12:26:09.871971154 +0000 2024-03-21T12:29:36.491 DEBUG:teuthology.orchestra.run.smithi022:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-03-21T12:29:36.556 INFO:teuthology.orchestra.run.smithi022.stderr:1+0 records in 2024-03-21T12:29:36.556 INFO:teuthology.orchestra.run.smithi022.stderr:1+0 records out 2024-03-21T12:29:36.556 INFO:teuthology.orchestra.run.smithi022.stderr:512 bytes copied, 8.4882e-05 s, 6.0 MB/s 2024-03-21T12:29:36.557 DEBUG:teuthology.orchestra.run.smithi022:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-03-21T12:29:36.613 DEBUG:teuthology.orchestra.run.smithi022:> stat /dev/vg_nvme/lv_4 2024-03-21T12:29:36.668 INFO:teuthology.orchestra.run.smithi022.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-03-21T12:29:36.668 INFO:teuthology.orchestra.run.smithi022.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-21T12:29:36.668 INFO:teuthology.orchestra.run.smithi022.stdout:Device: 5h/5d Inode: 722 Links: 1 2024-03-21T12:29:36.668 INFO:teuthology.orchestra.run.smithi022.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-21T12:29:36.668 INFO:teuthology.orchestra.run.smithi022.stdout:Context: system_u:object_r:device_t:s0 2024-03-21T12:29:36.668 INFO:teuthology.orchestra.run.smithi022.stdout:Access: 2024-03-21 12:28:55.874504790 +0000 2024-03-21T12:29:36.668 INFO:teuthology.orchestra.run.smithi022.stdout:Modify: 2024-03-21 12:26:10.217966587 +0000 2024-03-21T12:29:36.668 INFO:teuthology.orchestra.run.smithi022.stdout:Change: 2024-03-21 12:26:10.217966587 +0000 2024-03-21T12:29:36.668 INFO:teuthology.orchestra.run.smithi022.stdout: Birth: 2024-03-21 12:26:10.217966587 +0000 2024-03-21T12:29:36.668 DEBUG:teuthology.orchestra.run.smithi022:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-03-21T12:29:36.733 INFO:teuthology.orchestra.run.smithi022.stderr:1+0 records in 2024-03-21T12:29:36.734 INFO:teuthology.orchestra.run.smithi022.stderr:1+0 records out 2024-03-21T12:29:36.734 INFO:teuthology.orchestra.run.smithi022.stderr:512 bytes copied, 0.00010012 s, 5.1 MB/s 2024-03-21T12:29:36.735 DEBUG:teuthology.orchestra.run.smithi022:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-03-21T12:29:36.790 DEBUG:teuthology.orchestra.run.smithi189:> set -ex 2024-03-21T12:29:36.790 DEBUG:teuthology.orchestra.run.smithi189:> dd if=/scratch_devs of=/dev/stdout 2024-03-21T12:29:36.809 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-03-21T12:29:36.809 DEBUG:teuthology.orchestra.run.smithi189:> stat /dev/vg_nvme/lv_1 2024-03-21T12:29:36.868 INFO:teuthology.orchestra.run.smithi189.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-03-21T12:29:36.868 INFO:teuthology.orchestra.run.smithi189.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-21T12:29:36.868 INFO:teuthology.orchestra.run.smithi189.stdout:Device: 5h/5d Inode: 666 Links: 1 2024-03-21T12:29:36.868 INFO:teuthology.orchestra.run.smithi189.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-21T12:29:36.868 INFO:teuthology.orchestra.run.smithi189.stdout:Context: system_u:object_r:device_t:s0 2024-03-21T12:29:36.868 INFO:teuthology.orchestra.run.smithi189.stdout:Access: 2024-03-21 12:29:30.599821908 +0000 2024-03-21T12:29:36.868 INFO:teuthology.orchestra.run.smithi189.stdout:Modify: 2024-03-21 12:26:06.629353549 +0000 2024-03-21T12:29:36.868 INFO:teuthology.orchestra.run.smithi189.stdout:Change: 2024-03-21 12:26:06.629353549 +0000 2024-03-21T12:29:36.868 INFO:teuthology.orchestra.run.smithi189.stdout: Birth: 2024-03-21 12:26:06.629353549 +0000 2024-03-21T12:29:36.868 DEBUG:teuthology.orchestra.run.smithi189:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-03-21T12:29:36.937 INFO:teuthology.orchestra.run.smithi189.stderr:1+0 records in 2024-03-21T12:29:36.937 INFO:teuthology.orchestra.run.smithi189.stderr:1+0 records out 2024-03-21T12:29:36.937 INFO:teuthology.orchestra.run.smithi189.stderr:512 bytes copied, 0.000233486 s, 2.2 MB/s 2024-03-21T12:29:36.938 DEBUG:teuthology.orchestra.run.smithi189:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-03-21T12:29:36.997 DEBUG:teuthology.orchestra.run.smithi189:> stat /dev/vg_nvme/lv_2 2024-03-21T12:29:37.055 INFO:teuthology.orchestra.run.smithi189.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-03-21T12:29:37.055 INFO:teuthology.orchestra.run.smithi189.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-21T12:29:37.055 INFO:teuthology.orchestra.run.smithi189.stdout:Device: 5h/5d Inode: 684 Links: 1 2024-03-21T12:29:37.055 INFO:teuthology.orchestra.run.smithi189.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-21T12:29:37.055 INFO:teuthology.orchestra.run.smithi189.stdout:Context: system_u:object_r:device_t:s0 2024-03-21T12:29:37.055 INFO:teuthology.orchestra.run.smithi189.stdout:Access: 2024-03-21 12:29:30.599821908 +0000 2024-03-21T12:29:37.055 INFO:teuthology.orchestra.run.smithi189.stdout:Modify: 2024-03-21 12:26:06.968347002 +0000 2024-03-21T12:29:37.055 INFO:teuthology.orchestra.run.smithi189.stdout:Change: 2024-03-21 12:26:06.968347002 +0000 2024-03-21T12:29:37.055 INFO:teuthology.orchestra.run.smithi189.stdout: Birth: 2024-03-21 12:26:06.968347002 +0000 2024-03-21T12:29:37.056 DEBUG:teuthology.orchestra.run.smithi189:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-03-21T12:29:37.123 INFO:teuthology.orchestra.run.smithi189.stderr:1+0 records in 2024-03-21T12:29:37.123 INFO:teuthology.orchestra.run.smithi189.stderr:1+0 records out 2024-03-21T12:29:37.123 INFO:teuthology.orchestra.run.smithi189.stderr:512 bytes copied, 0.000207664 s, 2.5 MB/s 2024-03-21T12:29:37.124 DEBUG:teuthology.orchestra.run.smithi189:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-03-21T12:29:37.181 DEBUG:teuthology.orchestra.run.smithi189:> stat /dev/vg_nvme/lv_3 2024-03-21T12:29:37.237 INFO:teuthology.orchestra.run.smithi189.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-03-21T12:29:37.237 INFO:teuthology.orchestra.run.smithi189.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-21T12:29:37.237 INFO:teuthology.orchestra.run.smithi189.stdout:Device: 5h/5d Inode: 706 Links: 1 2024-03-21T12:29:37.237 INFO:teuthology.orchestra.run.smithi189.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-21T12:29:37.237 INFO:teuthology.orchestra.run.smithi189.stdout:Context: system_u:object_r:device_t:s0 2024-03-21T12:29:37.237 INFO:teuthology.orchestra.run.smithi189.stdout:Access: 2024-03-21 12:29:30.599821908 +0000 2024-03-21T12:29:37.237 INFO:teuthology.orchestra.run.smithi189.stdout:Modify: 2024-03-21 12:26:07.304340512 +0000 2024-03-21T12:29:37.237 INFO:teuthology.orchestra.run.smithi189.stdout:Change: 2024-03-21 12:26:07.304340512 +0000 2024-03-21T12:29:37.237 INFO:teuthology.orchestra.run.smithi189.stdout: Birth: 2024-03-21 12:26:07.304340512 +0000 2024-03-21T12:29:37.238 DEBUG:teuthology.orchestra.run.smithi189:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-03-21T12:29:37.303 INFO:teuthology.orchestra.run.smithi189.stderr:1+0 records in 2024-03-21T12:29:37.304 INFO:teuthology.orchestra.run.smithi189.stderr:1+0 records out 2024-03-21T12:29:37.304 INFO:teuthology.orchestra.run.smithi189.stderr:512 bytes copied, 0.000290344 s, 1.8 MB/s 2024-03-21T12:29:37.305 DEBUG:teuthology.orchestra.run.smithi189:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-03-21T12:29:37.361 DEBUG:teuthology.orchestra.run.smithi189:> stat /dev/vg_nvme/lv_4 2024-03-21T12:29:37.416 INFO:teuthology.orchestra.run.smithi189.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-03-21T12:29:37.416 INFO:teuthology.orchestra.run.smithi189.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-21T12:29:37.416 INFO:teuthology.orchestra.run.smithi189.stdout:Device: 5h/5d Inode: 723 Links: 1 2024-03-21T12:29:37.416 INFO:teuthology.orchestra.run.smithi189.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-21T12:29:37.416 INFO:teuthology.orchestra.run.smithi189.stdout:Context: system_u:object_r:device_t:s0 2024-03-21T12:29:37.416 INFO:teuthology.orchestra.run.smithi189.stdout:Access: 2024-03-21 12:29:30.600821889 +0000 2024-03-21T12:29:37.416 INFO:teuthology.orchestra.run.smithi189.stdout:Modify: 2024-03-21 12:26:07.638334061 +0000 2024-03-21T12:29:37.416 INFO:teuthology.orchestra.run.smithi189.stdout:Change: 2024-03-21 12:26:07.638334061 +0000 2024-03-21T12:29:37.416 INFO:teuthology.orchestra.run.smithi189.stdout: Birth: 2024-03-21 12:26:07.638334061 +0000 2024-03-21T12:29:37.417 DEBUG:teuthology.orchestra.run.smithi189:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-03-21T12:29:37.482 INFO:teuthology.orchestra.run.smithi189.stderr:1+0 records in 2024-03-21T12:29:37.482 INFO:teuthology.orchestra.run.smithi189.stderr:1+0 records out 2024-03-21T12:29:37.482 INFO:teuthology.orchestra.run.smithi189.stderr:512 bytes copied, 0.000211432 s, 2.4 MB/s 2024-03-21T12:29:37.483 DEBUG:teuthology.orchestra.run.smithi189:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-03-21T12:29:37.542 INFO:tasks.cephadm:Deploying osd.0 on smithi022 with /dev/vg_nvme/lv_4... 2024-03-21T12:29:37.543 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-03-21T12:29:37.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:37 smithi022 ceph-mon[24777]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-21T12:29:37.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:37 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:29:38.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:29:37 smithi189 ceph-mon[27080]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-21T12:29:38.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:29:37 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:29:39.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:38 smithi022 ceph-mon[24777]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-21T12:29:39.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:29:38 smithi189 ceph-mon[27080]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-21T12:29:41.164 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:40 smithi022 ceph-mon[24777]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-21T12:29:41.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:29:40 smithi189 ceph-mon[27080]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-21T12:29:42.019 INFO:teuthology.orchestra.run.smithi022.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-03-21T12:29:42.019 INFO:teuthology.orchestra.run.smithi022.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-03-21T12:29:42.019 INFO:teuthology.orchestra.run.smithi022.stderr: stderr: 10+0 records in 2024-03-21T12:29:42.019 INFO:teuthology.orchestra.run.smithi022.stderr:10+0 records out 2024-03-21T12:29:42.019 INFO:teuthology.orchestra.run.smithi022.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.00828546 s, 1.3 GB/s 2024-03-21T12:29:42.020 INFO:teuthology.orchestra.run.smithi022.stderr:--> Zapping successful for: 2024-03-21T12:29:43.214 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:42 smithi022 ceph-mon[24777]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-21T12:29:43.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:29:42 smithi189 ceph-mon[27080]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-21T12:29:43.835 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -- ceph orch daemon add osd smithi022:vg_nvme/lv_4 2024-03-21T12:29:45.016 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:44 smithi022 ceph-mon[24777]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-21T12:29:45.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:29:44 smithi189 ceph-mon[27080]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-21T12:29:48.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:29:47 smithi189 ceph-mon[27080]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-21T12:29:48.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:29:47 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-21T12:29:48.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:29:47 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-21T12:29:48.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:29:47 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:29:48.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:29:47 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:29:48.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:29:47 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:29:48.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:29:47 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:29:48.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:29:47 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:29:48.087 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:47 smithi022 ceph-mon[24777]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-21T12:29:48.087 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:47 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-21T12:29:48.087 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:47 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-21T12:29:48.087 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:47 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:29:48.087 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:47 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:29:48.088 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:47 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:29:48.088 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:47 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:29:48.088 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:47 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:29:48.964 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:48 smithi022 ceph-mon[24777]: from='client.14236 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi022:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:29:49.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:29:48 smithi189 ceph-mon[27080]: from='client.14236 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi022:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:29:50.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:29:49 smithi189 ceph-mon[27080]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-21T12:29:50.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:29:49 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:29:50.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:29:49 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:29:50.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:49 smithi022 ceph-mon[24777]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-21T12:29:50.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:49 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:29:50.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:49 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:29:51.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:50 smithi022 ceph-mon[24777]: Updating smithi189:/etc/ceph/ceph.conf 2024-03-21T12:29:51.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:50 smithi022 ceph-mon[24777]: Updating smithi189:/etc/ceph/ceph.client.admin.keyring 2024-03-21T12:29:51.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:50 smithi022 ceph-mon[24777]: from='client.? 172.21.15.22:0/409845425' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b91e0a05-db30-4947-ae26-b5e95e7bf8aa"}]: dispatch 2024-03-21T12:29:51.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:50 smithi022 ceph-mon[24777]: from='client.? 172.21.15.22:0/409845425' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b91e0a05-db30-4947-ae26-b5e95e7bf8aa"}]': finished 2024-03-21T12:29:51.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:50 smithi022 ceph-mon[24777]: osdmap e6: 1 total, 0 up, 1 in 2024-03-21T12:29:51.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:50 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-21T12:29:51.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:29:50 smithi189 ceph-mon[27080]: Updating smithi189:/etc/ceph/ceph.conf 2024-03-21T12:29:51.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:29:50 smithi189 ceph-mon[27080]: Updating smithi189:/etc/ceph/ceph.client.admin.keyring 2024-03-21T12:29:51.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:29:50 smithi189 ceph-mon[27080]: from='client.? 172.21.15.22:0/409845425' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b91e0a05-db30-4947-ae26-b5e95e7bf8aa"}]: dispatch 2024-03-21T12:29:51.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:29:50 smithi189 ceph-mon[27080]: from='client.? 172.21.15.22:0/409845425' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b91e0a05-db30-4947-ae26-b5e95e7bf8aa"}]': finished 2024-03-21T12:29:51.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:29:50 smithi189 ceph-mon[27080]: osdmap e6: 1 total, 0 up, 1 in 2024-03-21T12:29:51.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:29:50 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-21T12:29:52.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:51 smithi022 ceph-mon[24777]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-21T12:29:52.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:51 smithi022 ceph-mon[24777]: from='client.? 172.21.15.22:0/473157333' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-21T12:29:52.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:29:51 smithi189 ceph-mon[27080]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-21T12:29:52.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:29:51 smithi189 ceph-mon[27080]: from='client.? 172.21.15.22:0/473157333' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-21T12:29:53.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:29:52 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:29:53.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:29:52 smithi189 ceph-mon[27080]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-21T12:29:53.317 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:52 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:29:53.317 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:52 smithi022 ceph-mon[24777]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-21T12:29:55.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:55 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:29:55.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:55 smithi022 ceph-mon[24777]: Updating smithi022:/etc/ceph/ceph.conf 2024-03-21T12:29:55.595 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:55 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:29:55.595 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:55 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:29:55.595 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:55 smithi022 ceph-mon[24777]: Reconfiguring mon.smithi022 (unknown last config time)... 2024-03-21T12:29:55.595 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:55 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-21T12:29:55.595 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:55 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-21T12:29:55.595 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:55 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:29:55.595 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:55 smithi022 ceph-mon[24777]: Reconfiguring daemon mon.smithi022 on smithi022 2024-03-21T12:29:55.595 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:55 smithi022 ceph-mon[24777]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-21T12:29:55.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:29:55 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:29:55.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:29:55 smithi189 ceph-mon[27080]: Updating smithi022:/etc/ceph/ceph.conf 2024-03-21T12:29:55.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:29:55 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:29:55.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:29:55 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:29:55.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:29:55 smithi189 ceph-mon[27080]: Reconfiguring mon.smithi022 (unknown last config time)... 2024-03-21T12:29:55.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:29:55 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-21T12:29:55.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:29:55 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-21T12:29:55.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:29:55 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:29:55.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:29:55 smithi189 ceph-mon[27080]: Reconfiguring daemon mon.smithi022 on smithi022 2024-03-21T12:29:55.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:29:55 smithi189 ceph-mon[27080]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-21T12:29:57.677 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:57 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:29:57.677 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:57 smithi022 ceph-mon[24777]: Reconfiguring mgr.smithi022.fektzs (unknown last config time)... 2024-03-21T12:29:57.677 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:57 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi022.fektzs", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-21T12:29:57.677 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:57 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-21T12:29:57.677 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:57 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:29:57.677 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:57 smithi022 ceph-mon[24777]: Reconfiguring daemon mgr.smithi022.fektzs on smithi022 2024-03-21T12:29:57.677 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:57 smithi022 ceph-mon[24777]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-21T12:29:57.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:29:57 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:29:57.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:29:57 smithi189 ceph-mon[27080]: Reconfiguring mgr.smithi022.fektzs (unknown last config time)... 2024-03-21T12:29:57.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:29:57 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi022.fektzs", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-21T12:29:57.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:29:57 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-21T12:29:57.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:29:57 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:29:57.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:29:57 smithi189 ceph-mon[27080]: Reconfiguring daemon mgr.smithi022.fektzs on smithi022 2024-03-21T12:29:57.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:29:57 smithi189 ceph-mon[27080]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-21T12:29:58.717 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:58 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-03-21T12:29:58.718 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:58 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:29:58.718 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:58 smithi022 ceph-mon[24777]: Deploying daemon osd.0 on smithi022 2024-03-21T12:29:58.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:29:58 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-03-21T12:29:58.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:29:58 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:29:58.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:29:58 smithi189 ceph-mon[27080]: Deploying daemon osd.0 on smithi022 2024-03-21T12:30:00.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:29:59 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:30:00.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:29:59 smithi189 ceph-mon[27080]: Reconfiguring alertmanager.smithi022 (dependencies changed)... 2024-03-21T12:30:00.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:29:59 smithi189 ceph-mon[27080]: Reconfiguring daemon alertmanager.smithi022 on smithi022 2024-03-21T12:30:00.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:29:59 smithi189 ceph-mon[27080]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-21T12:30:00.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:59 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:30:00.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:59 smithi022 ceph-mon[24777]: Reconfiguring alertmanager.smithi022 (dependencies changed)... 2024-03-21T12:30:00.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:59 smithi022 ceph-mon[24777]: Reconfiguring daemon alertmanager.smithi022 on smithi022 2024-03-21T12:30:00.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:29:59 smithi022 ceph-mon[24777]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-21T12:30:01.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:00 smithi189 ceph-mon[27080]: overall HEALTH_OK 2024-03-21T12:30:01.039 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:00 smithi022 ceph-mon[24777]: overall HEALTH_OK 2024-03-21T12:30:02.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:01 smithi189 ceph-mon[27080]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-21T12:30:02.042 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:01 smithi022 ceph-mon[24777]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-21T12:30:03.045 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:02 smithi022 ceph-mon[24777]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-21T12:30:03.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:02 smithi189 ceph-mon[27080]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-21T12:30:05.007 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:04 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:30:05.007 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:04 smithi022 ceph-mon[24777]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-21T12:30:05.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:04 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:30:05.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:04 smithi189 ceph-mon[27080]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-21T12:30:08.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:07 smithi022 ceph-mon[24777]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-21T12:30:08.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:07 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi022.fektzs/mirror_snapshot_schedule"}]: dispatch 2024-03-21T12:30:08.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:07 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi022.fektzs/trash_purge_schedule"}]: dispatch 2024-03-21T12:30:08.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:07 smithi022 ceph-mon[24777]: from='osd.0 [v2:172.21.15.22:6802/1066511009,v1:172.21.15.22:6803/1066511009]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-03-21T12:30:08.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:07 smithi189 ceph-mon[27080]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-21T12:30:08.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:07 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi022.fektzs/mirror_snapshot_schedule"}]: dispatch 2024-03-21T12:30:08.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:07 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi022.fektzs/trash_purge_schedule"}]: dispatch 2024-03-21T12:30:08.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:07 smithi189 ceph-mon[27080]: from='osd.0 [v2:172.21.15.22:6802/1066511009,v1:172.21.15.22:6803/1066511009]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-03-21T12:30:08.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:08 smithi022 ceph-mon[24777]: from='osd.0 [v2:172.21.15.22:6802/1066511009,v1:172.21.15.22:6803/1066511009]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-03-21T12:30:08.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:08 smithi022 ceph-mon[24777]: osdmap e7: 1 total, 0 up, 1 in 2024-03-21T12:30:08.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:08 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-21T12:30:08.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:08 smithi022 ceph-mon[24777]: from='osd.0 [v2:172.21.15.22:6802/1066511009,v1:172.21.15.22:6803/1066511009]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]: dispatch 2024-03-21T12:30:09.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:08 smithi189 ceph-mon[27080]: from='osd.0 [v2:172.21.15.22:6802/1066511009,v1:172.21.15.22:6803/1066511009]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-03-21T12:30:09.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:08 smithi189 ceph-mon[27080]: osdmap e7: 1 total, 0 up, 1 in 2024-03-21T12:30:09.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:08 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-21T12:30:09.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:08 smithi189 ceph-mon[27080]: from='osd.0 [v2:172.21.15.22:6802/1066511009,v1:172.21.15.22:6803/1066511009]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]: dispatch 2024-03-21T12:30:10.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:09 smithi022 ceph-mon[24777]: from='osd.0 [v2:172.21.15.22:6802/1066511009,v1:172.21.15.22:6803/1066511009]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]': finished 2024-03-21T12:30:10.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:09 smithi022 ceph-mon[24777]: osdmap e8: 1 total, 0 up, 1 in 2024-03-21T12:30:10.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:09 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-21T12:30:10.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:09 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-21T12:30:10.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:09 smithi022 ceph-mon[24777]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-21T12:30:10.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:09 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:30:10.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:09 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi022", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-03-21T12:30:10.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:09 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:30:10.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:09 smithi189 ceph-mon[27080]: from='osd.0 [v2:172.21.15.22:6802/1066511009,v1:172.21.15.22:6803/1066511009]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]': finished 2024-03-21T12:30:10.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:09 smithi189 ceph-mon[27080]: osdmap e8: 1 total, 0 up, 1 in 2024-03-21T12:30:10.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:09 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-21T12:30:10.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:09 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-21T12:30:10.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:09 smithi189 ceph-mon[27080]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-21T12:30:10.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:09 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:30:10.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:09 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi022", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-03-21T12:30:10.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:09 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:30:10.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:10 smithi022 ceph-mon[24777]: purged_snaps scrub starts 2024-03-21T12:30:10.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:10 smithi022 ceph-mon[24777]: purged_snaps scrub ok 2024-03-21T12:30:10.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:10 smithi022 ceph-mon[24777]: Reconfiguring crash.smithi022 (monmap changed)... 2024-03-21T12:30:10.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:10 smithi022 ceph-mon[24777]: Reconfiguring daemon crash.smithi022 on smithi022 2024-03-21T12:30:10.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:10 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-21T12:30:10.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:10 smithi022 ceph-mon[24777]: osd.0 [v2:172.21.15.22:6802/1066511009,v1:172.21.15.22:6803/1066511009] boot 2024-03-21T12:30:10.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:10 smithi022 ceph-mon[24777]: osdmap e9: 1 total, 1 up, 1 in 2024-03-21T12:30:10.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:10 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-21T12:30:10.962 INFO:teuthology.orchestra.run.smithi022.stdout:Created osd(s) 0 on host 'smithi022' 2024-03-21T12:30:11.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:10 smithi189 ceph-mon[27080]: purged_snaps scrub starts 2024-03-21T12:30:11.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:10 smithi189 ceph-mon[27080]: purged_snaps scrub ok 2024-03-21T12:30:11.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:10 smithi189 ceph-mon[27080]: Reconfiguring crash.smithi022 (monmap changed)... 2024-03-21T12:30:11.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:10 smithi189 ceph-mon[27080]: Reconfiguring daemon crash.smithi022 on smithi022 2024-03-21T12:30:11.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:10 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-21T12:30:11.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:10 smithi189 ceph-mon[27080]: osd.0 [v2:172.21.15.22:6802/1066511009,v1:172.21.15.22:6803/1066511009] boot 2024-03-21T12:30:11.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:10 smithi189 ceph-mon[27080]: osdmap e9: 1 total, 1 up, 1 in 2024-03-21T12:30:11.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:10 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-21T12:30:11.771 DEBUG:teuthology.orchestra.run.smithi022:osd.0> sudo journalctl -f -n 0 -u ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715@osd.0.service 2024-03-21T12:30:11.773 INFO:tasks.cephadm:Deploying osd.1 on smithi022 with /dev/vg_nvme/lv_3... 2024-03-21T12:30:11.774 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-03-21T12:30:12.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:11 smithi022 ceph-mon[24777]: osdmap e10: 1 total, 1 up, 1 in 2024-03-21T12:30:12.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:11 smithi022 ceph-mon[24777]: pgmap v30: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-03-21T12:30:12.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:11 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:30:12.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:11 smithi189 ceph-mon[27080]: osdmap e10: 1 total, 1 up, 1 in 2024-03-21T12:30:12.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:11 smithi189 ceph-mon[27080]: pgmap v30: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-03-21T12:30:12.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:11 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:30:13.704 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:13 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:30:13.704 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:13 smithi022 ceph-mon[24777]: Reconfiguring grafana.smithi022 (dependencies changed)... 2024-03-21T12:30:13.705 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:13 smithi022 ceph-mon[24777]: Reconfiguring daemon grafana.smithi022 on smithi022 2024-03-21T12:30:13.705 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:13 smithi022 ceph-mon[24777]: pgmap v31: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-03-21T12:30:14.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:13 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:30:14.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:13 smithi189 ceph-mon[27080]: Reconfiguring grafana.smithi022 (dependencies changed)... 2024-03-21T12:30:14.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:13 smithi189 ceph-mon[27080]: Reconfiguring daemon grafana.smithi022 on smithi022 2024-03-21T12:30:14.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:13 smithi189 ceph-mon[27080]: pgmap v31: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-03-21T12:30:14.759 INFO:teuthology.orchestra.run.smithi022.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-03-21T12:30:14.760 INFO:teuthology.orchestra.run.smithi022.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-03-21T12:30:14.760 INFO:teuthology.orchestra.run.smithi022.stderr: stderr: 10+0 records in 2024-03-21T12:30:14.760 INFO:teuthology.orchestra.run.smithi022.stderr:10+0 records out 2024-03-21T12:30:14.760 INFO:teuthology.orchestra.run.smithi022.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.00844987 s, 1.2 GB/s 2024-03-21T12:30:14.760 INFO:teuthology.orchestra.run.smithi022.stderr:--> Zapping successful for: 2024-03-21T12:30:15.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:14 smithi022 ceph-mon[24777]: pgmap v32: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-03-21T12:30:15.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:14 smithi189 ceph-mon[27080]: pgmap v32: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-03-21T12:30:16.313 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -- ceph orch daemon add osd smithi022:vg_nvme/lv_3 2024-03-21T12:30:18.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:17 smithi022 ceph-mon[24777]: pgmap v33: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-03-21T12:30:18.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:17 smithi189 ceph-mon[27080]: pgmap v33: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-03-21T12:30:19.078 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:18 smithi022 ceph-mon[24777]: pgmap v34: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-03-21T12:30:19.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:18 smithi189 ceph-mon[27080]: pgmap v34: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-03-21T12:30:19.935 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:19 smithi022 ceph-mon[24777]: from='client.14254 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi022:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:30:19.935 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:19 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-21T12:30:19.936 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:19 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-21T12:30:19.936 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:19 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:30:19.936 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:19 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:30:19.936 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:19 smithi022 ceph-mon[24777]: Reconfiguring crash.smithi189 (monmap changed)... 2024-03-21T12:30:19.936 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:19 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi189", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-03-21T12:30:19.936 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:19 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:30:19.936 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:19 smithi022 ceph-mon[24777]: Reconfiguring daemon crash.smithi189 on smithi189 2024-03-21T12:30:20.219 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:19 smithi189 ceph-mon[27080]: from='client.14254 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi022:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:30:20.220 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:19 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-21T12:30:20.220 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:19 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-21T12:30:20.220 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:19 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:30:20.220 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:19 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:30:20.220 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:19 smithi189 ceph-mon[27080]: Reconfiguring crash.smithi189 (monmap changed)... 2024-03-21T12:30:20.220 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:19 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi189", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-03-21T12:30:20.220 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:19 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:30:20.220 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:19 smithi189 ceph-mon[27080]: Reconfiguring daemon crash.smithi189 on smithi189 2024-03-21T12:30:20.959 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:20 smithi022 ceph-mon[24777]: pgmap v35: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-03-21T12:30:21.251 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:20 smithi189 ceph-mon[27080]: pgmap v35: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-03-21T12:30:22.143 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:21 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:30:22.143 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:21 smithi022 ceph-mon[24777]: Reconfiguring mgr.smithi189.cnxzvu (monmap changed)... 2024-03-21T12:30:22.143 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:21 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi189.cnxzvu", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-21T12:30:22.143 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:21 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-21T12:30:22.143 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:21 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:30:22.143 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:21 smithi022 ceph-mon[24777]: Reconfiguring daemon mgr.smithi189.cnxzvu on smithi189 2024-03-21T12:30:22.143 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:21 smithi022 ceph-mon[24777]: from='client.? 172.21.15.22:0/1226485035' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4f42c329-33f6-4c0e-9ce5-b8ecc4a3e6f6"}]: dispatch 2024-03-21T12:30:22.143 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:21 smithi022 ceph-mon[24777]: from='client.? 172.21.15.22:0/1226485035' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4f42c329-33f6-4c0e-9ce5-b8ecc4a3e6f6"}]': finished 2024-03-21T12:30:22.144 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:21 smithi022 ceph-mon[24777]: osdmap e11: 2 total, 1 up, 2 in 2024-03-21T12:30:22.144 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:21 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-21T12:30:22.203 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:21 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:30:22.203 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:21 smithi189 ceph-mon[27080]: Reconfiguring mgr.smithi189.cnxzvu (monmap changed)... 2024-03-21T12:30:22.203 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:21 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi189.cnxzvu", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-21T12:30:22.203 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:21 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-21T12:30:22.204 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:21 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:30:22.204 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:21 smithi189 ceph-mon[27080]: Reconfiguring daemon mgr.smithi189.cnxzvu on smithi189 2024-03-21T12:30:22.204 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:21 smithi189 ceph-mon[27080]: from='client.? 172.21.15.22:0/1226485035' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4f42c329-33f6-4c0e-9ce5-b8ecc4a3e6f6"}]: dispatch 2024-03-21T12:30:22.204 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:21 smithi189 ceph-mon[27080]: from='client.? 172.21.15.22:0/1226485035' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4f42c329-33f6-4c0e-9ce5-b8ecc4a3e6f6"}]': finished 2024-03-21T12:30:22.204 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:21 smithi189 ceph-mon[27080]: osdmap e11: 2 total, 1 up, 2 in 2024-03-21T12:30:22.204 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:21 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-21T12:30:23.284 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:22 smithi189 ceph-mon[27080]: from='client.? 172.21.15.22:0/595622259' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-21T12:30:23.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:22 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:30:23.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:22 smithi189 ceph-mon[27080]: Reconfiguring mon.smithi189 (monmap changed)... 2024-03-21T12:30:23.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:22 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-21T12:30:23.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:22 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-21T12:30:23.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:22 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:30:23.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:22 smithi189 ceph-mon[27080]: Reconfiguring daemon mon.smithi189 on smithi189 2024-03-21T12:30:23.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:22 smithi189 ceph-mon[27080]: pgmap v37: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-03-21T12:30:23.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:22 smithi022 ceph-mon[24777]: from='client.? 172.21.15.22:0/595622259' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-21T12:30:23.345 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:22 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:30:23.345 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:22 smithi022 ceph-mon[24777]: Reconfiguring mon.smithi189 (monmap changed)... 2024-03-21T12:30:23.345 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:22 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-21T12:30:23.345 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:22 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-21T12:30:23.345 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:22 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:30:23.345 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:22 smithi022 ceph-mon[24777]: Reconfiguring daemon mon.smithi189 on smithi189 2024-03-21T12:30:23.345 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:22 smithi022 ceph-mon[24777]: pgmap v37: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-03-21T12:30:24.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:24 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:30:24.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:24 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-21T12:30:24.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:24 smithi022 ceph-mon[24777]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-21T12:30:24.996 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:24 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:30:24.996 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:24 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-21T12:30:24.996 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:24 smithi189 ceph-mon[27080]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-21T12:30:25.674 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:25 smithi022 ceph-mon[24777]: pgmap v38: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-03-21T12:30:26.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:25 smithi189 ceph-mon[27080]: pgmap v38: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-03-21T12:30:28.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:27 smithi022 ceph-mon[24777]: pgmap v39: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-03-21T12:30:28.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:27 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-03-21T12:30:28.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:27 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:30:28.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:27 smithi189 ceph-mon[27080]: pgmap v39: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-03-21T12:30:28.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:27 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-03-21T12:30:28.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:27 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:30:29.093 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:28 smithi022 ceph-mon[24777]: Deploying daemon osd.1 on smithi022 2024-03-21T12:30:29.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:28 smithi189 ceph-mon[27080]: Deploying daemon osd.1 on smithi022 2024-03-21T12:30:30.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:29 smithi022 ceph-mon[24777]: pgmap v40: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-03-21T12:30:30.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:29 smithi189 ceph-mon[27080]: pgmap v40: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-03-21T12:30:31.170 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:30 smithi022 ceph-mon[24777]: pgmap v41: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-03-21T12:30:31.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:30 smithi189 ceph-mon[27080]: pgmap v41: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-03-21T12:30:34.025 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:33 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:30:34.025 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:33 smithi022 ceph-mon[24777]: pgmap v42: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-03-21T12:30:34.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:33 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:30:34.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:33 smithi189 ceph-mon[27080]: pgmap v42: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-03-21T12:30:35.081 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:34 smithi022 ceph-mon[24777]: pgmap v43: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-03-21T12:30:35.232 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:34 smithi189 ceph-mon[27080]: pgmap v43: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-03-21T12:30:35.375 INFO:teuthology.orchestra.run.smithi022.stdout:Created osd(s) 1 on host 'smithi022' 2024-03-21T12:30:36.017 DEBUG:teuthology.orchestra.run.smithi022:osd.1> sudo journalctl -f -n 0 -u ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715@osd.1.service 2024-03-21T12:30:36.020 INFO:tasks.cephadm:Deploying osd.2 on smithi022 with /dev/vg_nvme/lv_2... 2024-03-21T12:30:36.020 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-03-21T12:30:36.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:36 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:30:36.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:36 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.22:9093"}]: dispatch 2024-03-21T12:30:36.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:36 smithi189 ceph-mon[27080]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.22:9093"}]: dispatch 2024-03-21T12:30:36.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:36 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:30:36.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:36 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-21T12:30:36.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:36 smithi189 ceph-mon[27080]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-21T12:30:36.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:36 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.22:3000"}]: dispatch 2024-03-21T12:30:36.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:36 smithi189 ceph-mon[27080]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.22:3000"}]: dispatch 2024-03-21T12:30:36.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:36 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:30:36.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:36 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-21T12:30:36.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:36 smithi189 ceph-mon[27080]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-21T12:30:36.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:36 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.22:9095"}]: dispatch 2024-03-21T12:30:36.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:36 smithi189 ceph-mon[27080]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.22:9095"}]: dispatch 2024-03-21T12:30:36.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:36 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:30:36.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:36 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:30:36.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:36 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:30:36.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:36 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:30:36.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:36 smithi189 ceph-mon[27080]: from='osd.1 [v2:172.21.15.22:6810/3661439817,v1:172.21.15.22:6811/3661439817]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-03-21T12:30:36.656 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:36 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:30:36.656 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:36 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.22:9093"}]: dispatch 2024-03-21T12:30:36.656 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:36 smithi022 ceph-mon[24777]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.22:9093"}]: dispatch 2024-03-21T12:30:36.656 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:36 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:30:36.656 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:36 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-21T12:30:36.656 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:36 smithi022 ceph-mon[24777]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-21T12:30:36.656 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:36 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.22:3000"}]: dispatch 2024-03-21T12:30:36.656 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:36 smithi022 ceph-mon[24777]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.22:3000"}]: dispatch 2024-03-21T12:30:36.656 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:36 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:30:36.657 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:36 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-21T12:30:36.657 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:36 smithi022 ceph-mon[24777]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-21T12:30:36.657 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:36 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.22:9095"}]: dispatch 2024-03-21T12:30:36.657 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:36 smithi022 ceph-mon[24777]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.22:9095"}]: dispatch 2024-03-21T12:30:36.657 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:36 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:30:36.657 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:36 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:30:36.657 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:36 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:30:36.657 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:36 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:30:36.657 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:36 smithi022 ceph-mon[24777]: from='osd.1 [v2:172.21.15.22:6810/3661439817,v1:172.21.15.22:6811/3661439817]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-03-21T12:30:37.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:37 smithi022 ceph-mon[24777]: from='osd.1 [v2:172.21.15.22:6810/3661439817,v1:172.21.15.22:6811/3661439817]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-03-21T12:30:37.595 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:37 smithi022 ceph-mon[24777]: osdmap e12: 2 total, 1 up, 2 in 2024-03-21T12:30:37.595 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:37 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-21T12:30:37.595 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:37 smithi022 ceph-mon[24777]: from='osd.1 [v2:172.21.15.22:6810/3661439817,v1:172.21.15.22:6811/3661439817]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]: dispatch 2024-03-21T12:30:37.595 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:37 smithi022 ceph-mon[24777]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-03-21T12:30:37.595 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:37 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:30:37.595 INFO:journalctl@ceph.osd.1.smithi022.stdout:Mar 21 12:30:37 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-1[38048]: 2024-03-21T12:30:37.415+0000 7f0cd7b55700 -1 osd.1 0 waiting for initial osdmap 2024-03-21T12:30:37.595 INFO:journalctl@ceph.osd.1.smithi022.stdout:Mar 21 12:30:37 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-1[38048]: 2024-03-21T12:30:37.419+0000 7f0cd44f0700 -1 osd.1 13 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-21T12:30:37.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:37 smithi189 ceph-mon[27080]: from='osd.1 [v2:172.21.15.22:6810/3661439817,v1:172.21.15.22:6811/3661439817]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-03-21T12:30:37.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:37 smithi189 ceph-mon[27080]: osdmap e12: 2 total, 1 up, 2 in 2024-03-21T12:30:37.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:37 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-21T12:30:37.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:37 smithi189 ceph-mon[27080]: from='osd.1 [v2:172.21.15.22:6810/3661439817,v1:172.21.15.22:6811/3661439817]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]: dispatch 2024-03-21T12:30:37.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:37 smithi189 ceph-mon[27080]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-03-21T12:30:37.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:37 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:30:38.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:38 smithi022 ceph-mon[24777]: from='osd.1 [v2:172.21.15.22:6810/3661439817,v1:172.21.15.22:6811/3661439817]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]': finished 2024-03-21T12:30:38.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:38 smithi022 ceph-mon[24777]: osdmap e13: 2 total, 1 up, 2 in 2024-03-21T12:30:38.595 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:38 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-21T12:30:38.595 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:38 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-21T12:30:38.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:38 smithi189 ceph-mon[27080]: from='osd.1 [v2:172.21.15.22:6810/3661439817,v1:172.21.15.22:6811/3661439817]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]': finished 2024-03-21T12:30:38.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:38 smithi189 ceph-mon[27080]: osdmap e13: 2 total, 1 up, 2 in 2024-03-21T12:30:38.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:38 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-21T12:30:38.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:38 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-21T12:30:39.372 INFO:teuthology.orchestra.run.smithi022.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-03-21T12:30:39.373 INFO:teuthology.orchestra.run.smithi022.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-03-21T12:30:39.373 INFO:teuthology.orchestra.run.smithi022.stderr: stderr: 10+0 records in 2024-03-21T12:30:39.373 INFO:teuthology.orchestra.run.smithi022.stderr:10+0 records out 2024-03-21T12:30:39.373 INFO:teuthology.orchestra.run.smithi022.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.00932393 s, 1.1 GB/s 2024-03-21T12:30:39.373 INFO:teuthology.orchestra.run.smithi022.stderr:--> Zapping successful for: 2024-03-21T12:30:39.651 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:39 smithi022 ceph-mon[24777]: purged_snaps scrub starts 2024-03-21T12:30:39.651 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:39 smithi022 ceph-mon[24777]: purged_snaps scrub ok 2024-03-21T12:30:39.651 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:39 smithi022 ceph-mon[24777]: osd.1 [v2:172.21.15.22:6810/3661439817,v1:172.21.15.22:6811/3661439817] boot 2024-03-21T12:30:39.652 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:39 smithi022 ceph-mon[24777]: osdmap e14: 2 total, 2 up, 2 in 2024-03-21T12:30:39.652 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:39 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-21T12:30:39.652 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:39 smithi022 ceph-mon[24777]: pgmap v48: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-03-21T12:30:39.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:39 smithi189 ceph-mon[27080]: purged_snaps scrub starts 2024-03-21T12:30:39.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:39 smithi189 ceph-mon[27080]: purged_snaps scrub ok 2024-03-21T12:30:39.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:39 smithi189 ceph-mon[27080]: osd.1 [v2:172.21.15.22:6810/3661439817,v1:172.21.15.22:6811/3661439817] boot 2024-03-21T12:30:39.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:39 smithi189 ceph-mon[27080]: osdmap e14: 2 total, 2 up, 2 in 2024-03-21T12:30:39.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:39 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-21T12:30:39.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:39 smithi189 ceph-mon[27080]: pgmap v48: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-03-21T12:30:40.023 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -- ceph orch daemon add osd smithi022:vg_nvme/lv_2 2024-03-21T12:30:40.746 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:40 smithi022 ceph-mon[24777]: osdmap e15: 2 total, 2 up, 2 in 2024-03-21T12:30:40.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:40 smithi189 ceph-mon[27080]: osdmap e15: 2 total, 2 up, 2 in 2024-03-21T12:30:41.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:41 smithi022 ceph-mon[24777]: pgmap v50: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-21T12:30:41.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:41 smithi189 ceph-mon[27080]: pgmap v50: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-21T12:30:43.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:42 smithi022 ceph-mon[24777]: pgmap v51: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-21T12:30:43.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:42 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:30:43.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:42 smithi189 ceph-mon[27080]: pgmap v51: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-21T12:30:43.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:42 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:30:44.081 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:43 smithi022 ceph-mon[24777]: from='client.14272 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi022:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:30:44.081 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:43 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-21T12:30:44.081 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:43 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-21T12:30:44.081 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:43 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:30:44.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:43 smithi189 ceph-mon[27080]: from='client.14272 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi022:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:30:44.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:43 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-21T12:30:44.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:43 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-21T12:30:44.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:43 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:30:44.978 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:44 smithi022 ceph-mon[24777]: pgmap v52: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-21T12:30:45.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:44 smithi189 ceph-mon[27080]: pgmap v52: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-21T12:30:47.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:47 smithi022 ceph-mon[24777]: Detected new or changed devices on smithi022 2024-03-21T12:30:47.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:47 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:30:47.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:47 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-21T12:30:47.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:47 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-21T12:30:47.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:47 smithi022 ceph-mon[24777]: Adjusting osd_memory_target on smithi022 to 6434M 2024-03-21T12:30:47.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:47 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:30:47.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:47 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:30:47.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:47 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:30:47.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:47 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:30:47.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:47 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:30:47.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:47 smithi022 ceph-mon[24777]: pgmap v53: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-21T12:30:48.034 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:47 smithi189 ceph-mon[27080]: Detected new or changed devices on smithi022 2024-03-21T12:30:48.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:47 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:30:48.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:47 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-21T12:30:48.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:47 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-21T12:30:48.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:47 smithi189 ceph-mon[27080]: Adjusting osd_memory_target on smithi022 to 6434M 2024-03-21T12:30:48.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:47 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:30:48.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:47 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:30:48.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:47 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:30:48.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:47 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:30:48.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:47 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:30:48.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:47 smithi189 ceph-mon[27080]: pgmap v53: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-21T12:30:49.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:48 smithi022 ceph-mon[24777]: from='client.? 172.21.15.22:0/2795710500' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1efc32fa-d9f6-496b-bc29-6aa4fce1a6ab"}]: dispatch 2024-03-21T12:30:49.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:48 smithi022 ceph-mon[24777]: from='client.? 172.21.15.22:0/2795710500' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1efc32fa-d9f6-496b-bc29-6aa4fce1a6ab"}]': finished 2024-03-21T12:30:49.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:48 smithi022 ceph-mon[24777]: osdmap e16: 3 total, 2 up, 3 in 2024-03-21T12:30:49.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:48 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-21T12:30:49.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:48 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:30:49.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:48 smithi022 ceph-mon[24777]: from='client.? 172.21.15.22:0/1171374597' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-21T12:30:49.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:48 smithi189 ceph-mon[27080]: from='client.? 172.21.15.22:0/2795710500' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1efc32fa-d9f6-496b-bc29-6aa4fce1a6ab"}]: dispatch 2024-03-21T12:30:49.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:48 smithi189 ceph-mon[27080]: from='client.? 172.21.15.22:0/2795710500' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1efc32fa-d9f6-496b-bc29-6aa4fce1a6ab"}]': finished 2024-03-21T12:30:49.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:48 smithi189 ceph-mon[27080]: osdmap e16: 3 total, 2 up, 3 in 2024-03-21T12:30:49.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:48 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-21T12:30:49.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:48 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:30:49.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:48 smithi189 ceph-mon[27080]: from='client.? 172.21.15.22:0/1171374597' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-21T12:30:50.057 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:49 smithi022 ceph-mon[24777]: pgmap v55: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-21T12:30:50.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:49 smithi189 ceph-mon[27080]: pgmap v55: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-21T12:30:51.216 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:50 smithi022 ceph-mon[24777]: pgmap v56: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-21T12:30:51.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:50 smithi189 ceph-mon[27080]: pgmap v56: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-21T12:30:52.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:52 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:30:53.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:52 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:30:53.694 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:53 smithi022 ceph-mon[24777]: pgmap v57: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-21T12:30:53.943 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:53 smithi189 ceph-mon[27080]: pgmap v57: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-21T12:30:54.953 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:54 smithi022 ceph-mon[24777]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-21T12:30:55.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:54 smithi189 ceph-mon[27080]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-21T12:30:56.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:55 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-03-21T12:30:56.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:55 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:30:56.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:55 smithi189 ceph-mon[27080]: Deploying daemon osd.2 on smithi022 2024-03-21T12:30:56.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:55 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-03-21T12:30:56.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:55 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:30:56.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:55 smithi022 ceph-mon[24777]: Deploying daemon osd.2 on smithi022 2024-03-21T12:30:57.924 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:57 smithi022 ceph-mon[24777]: Detected new or changed devices on smithi022 2024-03-21T12:30:57.924 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:57 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:30:57.924 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:57 smithi022 ceph-mon[24777]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-21T12:30:57.924 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:57 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:30:58.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:57 smithi189 ceph-mon[27080]: Detected new or changed devices on smithi022 2024-03-21T12:30:58.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:57 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:30:58.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:57 smithi189 ceph-mon[27080]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-21T12:30:58.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:57 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:30:59.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:30:59 smithi022 ceph-mon[24777]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-21T12:31:00.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:30:59 smithi189 ceph-mon[27080]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-21T12:31:01.834 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:01 smithi022 ceph-mon[24777]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-21T12:31:01.834 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:01 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:31:01.834 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:01 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:31:01.834 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:01 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:31:01.835 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:01 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:31:02.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:01 smithi189 ceph-mon[27080]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-21T12:31:02.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:01 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:31:02.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:01 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:31:02.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:01 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:31:02.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:01 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:31:03.048 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:02 smithi022 ceph-mon[24777]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-21T12:31:03.158 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:02 smithi189 ceph-mon[27080]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-21T12:31:04.664 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:04 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:31:04.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:04 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:31:05.320 INFO:teuthology.orchestra.run.smithi022.stdout:Created osd(s) 2 on host 'smithi022' 2024-03-21T12:31:05.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:05 smithi022 ceph-mon[24777]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-21T12:31:05.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:05 smithi022 ceph-mon[24777]: from='osd.2 [v2:172.21.15.22:6818/4170559295,v1:172.21.15.22:6819/4170559295]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-03-21T12:31:05.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:05 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:31:05.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:05 smithi189 ceph-mon[27080]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-21T12:31:05.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:05 smithi189 ceph-mon[27080]: from='osd.2 [v2:172.21.15.22:6818/4170559295,v1:172.21.15.22:6819/4170559295]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-03-21T12:31:05.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:05 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:31:05.943 DEBUG:teuthology.orchestra.run.smithi022:osd.2> sudo journalctl -f -n 0 -u ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715@osd.2.service 2024-03-21T12:31:05.945 INFO:tasks.cephadm:Deploying osd.3 on smithi022 with /dev/vg_nvme/lv_1... 2024-03-21T12:31:05.945 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-03-21T12:31:06.751 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:06 smithi022 ceph-mon[24777]: from='osd.2 [v2:172.21.15.22:6818/4170559295,v1:172.21.15.22:6819/4170559295]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-03-21T12:31:06.751 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:06 smithi022 ceph-mon[24777]: osdmap e17: 3 total, 2 up, 3 in 2024-03-21T12:31:06.751 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:06 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-21T12:31:06.751 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:06 smithi022 ceph-mon[24777]: from='osd.2 [v2:172.21.15.22:6818/4170559295,v1:172.21.15.22:6819/4170559295]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]: dispatch 2024-03-21T12:31:06.752 INFO:journalctl@ceph.osd.2.smithi022.stdout:Mar 21 12:31:06 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-2[41999]: 2024-03-21T12:31:06.454+0000 7f943c2f1700 -1 osd.2 0 waiting for initial osdmap 2024-03-21T12:31:06.752 INFO:journalctl@ceph.osd.2.smithi022.stdout:Mar 21 12:31:06 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-2[41999]: 2024-03-21T12:31:06.457+0000 7f9437c8a700 -1 osd.2 18 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-21T12:31:06.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:06 smithi189 ceph-mon[27080]: from='osd.2 [v2:172.21.15.22:6818/4170559295,v1:172.21.15.22:6819/4170559295]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-03-21T12:31:06.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:06 smithi189 ceph-mon[27080]: osdmap e17: 3 total, 2 up, 3 in 2024-03-21T12:31:06.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:06 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-21T12:31:06.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:06 smithi189 ceph-mon[27080]: from='osd.2 [v2:172.21.15.22:6818/4170559295,v1:172.21.15.22:6819/4170559295]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]: dispatch 2024-03-21T12:31:07.735 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:07 smithi022 ceph-mon[24777]: from='osd.2 [v2:172.21.15.22:6818/4170559295,v1:172.21.15.22:6819/4170559295]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]': finished 2024-03-21T12:31:07.735 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:07 smithi022 ceph-mon[24777]: osdmap e18: 3 total, 2 up, 3 in 2024-03-21T12:31:07.735 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:07 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-21T12:31:07.736 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:07 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-21T12:31:07.736 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:07 smithi022 ceph-mon[24777]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-21T12:31:07.736 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:07 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi022.fektzs/trash_purge_schedule"}]: dispatch 2024-03-21T12:31:07.736 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:07 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi022.fektzs/mirror_snapshot_schedule"}]: dispatch 2024-03-21T12:31:07.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:07 smithi189 ceph-mon[27080]: from='osd.2 [v2:172.21.15.22:6818/4170559295,v1:172.21.15.22:6819/4170559295]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]': finished 2024-03-21T12:31:07.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:07 smithi189 ceph-mon[27080]: osdmap e18: 3 total, 2 up, 3 in 2024-03-21T12:31:07.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:07 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-21T12:31:07.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:07 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-21T12:31:07.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:07 smithi189 ceph-mon[27080]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-21T12:31:07.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:07 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi022.fektzs/trash_purge_schedule"}]: dispatch 2024-03-21T12:31:07.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:07 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi022.fektzs/mirror_snapshot_schedule"}]: dispatch 2024-03-21T12:31:08.785 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:08 smithi022 ceph-mon[24777]: purged_snaps scrub starts 2024-03-21T12:31:08.785 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:08 smithi022 ceph-mon[24777]: purged_snaps scrub ok 2024-03-21T12:31:08.785 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:08 smithi022 ceph-mon[24777]: osd.2 [v2:172.21.15.22:6818/4170559295,v1:172.21.15.22:6819/4170559295] boot 2024-03-21T12:31:08.785 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:08 smithi022 ceph-mon[24777]: osdmap e19: 3 total, 3 up, 3 in 2024-03-21T12:31:08.785 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:08 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-21T12:31:08.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:08 smithi189 ceph-mon[27080]: purged_snaps scrub starts 2024-03-21T12:31:08.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:08 smithi189 ceph-mon[27080]: purged_snaps scrub ok 2024-03-21T12:31:08.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:08 smithi189 ceph-mon[27080]: osd.2 [v2:172.21.15.22:6818/4170559295,v1:172.21.15.22:6819/4170559295] boot 2024-03-21T12:31:08.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:08 smithi189 ceph-mon[27080]: osdmap e19: 3 total, 3 up, 3 in 2024-03-21T12:31:08.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:08 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-21T12:31:09.257 INFO:teuthology.orchestra.run.smithi022.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-03-21T12:31:09.257 INFO:teuthology.orchestra.run.smithi022.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-03-21T12:31:09.257 INFO:teuthology.orchestra.run.smithi022.stderr: stderr: 10+0 records in 2024-03-21T12:31:09.258 INFO:teuthology.orchestra.run.smithi022.stderr:10+0 records out 2024-03-21T12:31:09.258 INFO:teuthology.orchestra.run.smithi022.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.00934676 s, 1.1 GB/s 2024-03-21T12:31:09.258 INFO:teuthology.orchestra.run.smithi022.stderr:--> Zapping successful for: 2024-03-21T12:31:09.716 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:09 smithi022 ceph-mon[24777]: osdmap e20: 3 total, 3 up, 3 in 2024-03-21T12:31:09.716 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:09 smithi022 ceph-mon[24777]: pgmap v69: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-03-21T12:31:09.716 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:09 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-03-21T12:31:09.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:09 smithi189 ceph-mon[27080]: osdmap e20: 3 total, 3 up, 3 in 2024-03-21T12:31:09.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:09 smithi189 ceph-mon[27080]: pgmap v69: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-03-21T12:31:09.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:09 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-03-21T12:31:10.000 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -- ceph orch daemon add osd smithi022:vg_nvme/lv_1 2024-03-21T12:31:10.584 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:10 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-03-21T12:31:10.585 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:10 smithi022 ceph-mon[24777]: osdmap e21: 3 total, 3 up, 3 in 2024-03-21T12:31:10.585 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:10 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-03-21T12:31:10.585 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:10 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:31:10.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:10 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-03-21T12:31:10.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:10 smithi189 ceph-mon[27080]: osdmap e21: 3 total, 3 up, 3 in 2024-03-21T12:31:10.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:10 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-03-21T12:31:10.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:10 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:31:11.472 INFO:journalctl@ceph.osd.0.smithi022.stdout:Mar 21 12:31:11 smithi022 sudo[43494]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-03-21T12:31:11.472 INFO:journalctl@ceph.osd.0.smithi022.stdout:Mar 21 12:31:11 smithi022 sudo[43494]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-03-21T12:31:11.472 INFO:journalctl@ceph.osd.0.smithi022.stdout:Mar 21 12:31:11 smithi022 sudo[43494]: pam_unix(sudo:session): session closed for user root 2024-03-21T12:31:11.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:11 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-03-21T12:31:11.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:11 smithi189 ceph-mon[27080]: osdmap e22: 3 total, 3 up, 3 in 2024-03-21T12:31:11.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:11 smithi189 ceph-mon[27080]: pgmap v72: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-03-21T12:31:11.802 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:11 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-03-21T12:31:11.803 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:11 smithi022 ceph-mon[24777]: osdmap e22: 3 total, 3 up, 3 in 2024-03-21T12:31:11.803 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:11 smithi022 ceph-mon[24777]: pgmap v72: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-03-21T12:31:11.803 INFO:journalctl@ceph.osd.0.smithi022.stdout:Mar 21 12:31:11 smithi022 sudo[43501]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-03-21T12:31:11.803 INFO:journalctl@ceph.osd.0.smithi022.stdout:Mar 21 12:31:11 smithi022 sudo[43501]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-03-21T12:31:11.803 INFO:journalctl@ceph.osd.0.smithi022.stdout:Mar 21 12:31:11 smithi022 sudo[43501]: pam_unix(sudo:session): session closed for user root 2024-03-21T12:31:12.072 INFO:journalctl@ceph.osd.1.smithi022.stdout:Mar 21 12:31:11 smithi022 sudo[43540]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-03-21T12:31:12.072 INFO:journalctl@ceph.osd.1.smithi022.stdout:Mar 21 12:31:11 smithi022 sudo[43540]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-03-21T12:31:12.072 INFO:journalctl@ceph.osd.1.smithi022.stdout:Mar 21 12:31:11 smithi022 sudo[43540]: pam_unix(sudo:session): session closed for user root 2024-03-21T12:31:12.338 INFO:journalctl@ceph.osd.1.smithi022.stdout:Mar 21 12:31:12 smithi022 sudo[43584]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-03-21T12:31:12.338 INFO:journalctl@ceph.osd.1.smithi022.stdout:Mar 21 12:31:12 smithi022 sudo[43584]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-03-21T12:31:12.338 INFO:journalctl@ceph.osd.1.smithi022.stdout:Mar 21 12:31:12 smithi022 sudo[43584]: pam_unix(sudo:session): session closed for user root 2024-03-21T12:31:12.594 INFO:journalctl@ceph.osd.2.smithi022.stdout:Mar 21 12:31:12 smithi022 sudo[43610]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-03-21T12:31:12.595 INFO:journalctl@ceph.osd.2.smithi022.stdout:Mar 21 12:31:12 smithi022 sudo[43610]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-03-21T12:31:12.595 INFO:journalctl@ceph.osd.2.smithi022.stdout:Mar 21 12:31:12 smithi022 sudo[43610]: pam_unix(sudo:session): session closed for user root 2024-03-21T12:31:12.595 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:12 smithi022 ceph-mon[24777]: osdmap e23: 3 total, 3 up, 3 in 2024-03-21T12:31:12.595 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:12 smithi022 ceph-mon[24777]: mgrmap e19: smithi022.fektzs(active, since 2m), standbys: smithi189.cnxzvu 2024-03-21T12:31:12.595 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:12 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:31:12.595 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:12 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-21T12:31:12.595 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:12 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-21T12:31:12.595 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:12 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:31:12.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:12 smithi189 ceph-mon[27080]: osdmap e23: 3 total, 3 up, 3 in 2024-03-21T12:31:12.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:12 smithi189 ceph-mon[27080]: mgrmap e19: smithi022.fektzs(active, since 2m), standbys: smithi189.cnxzvu 2024-03-21T12:31:12.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:12 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:31:12.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:12 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-21T12:31:12.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:12 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-21T12:31:12.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:12 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:31:12.877 INFO:journalctl@ceph.osd.2.smithi022.stdout:Mar 21 12:31:12 smithi022 sudo[43651]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-03-21T12:31:12.877 INFO:journalctl@ceph.osd.2.smithi022.stdout:Mar 21 12:31:12 smithi022 sudo[43651]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-03-21T12:31:12.877 INFO:journalctl@ceph.osd.2.smithi022.stdout:Mar 21 12:31:12 smithi022 sudo[43651]: pam_unix(sudo:session): session closed for user root 2024-03-21T12:31:13.131 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:12 smithi022 sudo[43676]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-03-21T12:31:13.131 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:12 smithi022 sudo[43676]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-03-21T12:31:13.131 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:12 smithi022 sudo[43676]: pam_unix(sudo:session): session closed for user root 2024-03-21T12:31:13.390 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:13 smithi022 sudo[43684]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-03-21T12:31:13.390 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:13 smithi022 sudo[43684]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-03-21T12:31:13.390 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:13 smithi022 sudo[43684]: pam_unix(sudo:session): session closed for user root 2024-03-21T12:31:13.653 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:13 smithi189 sudo[29404]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-03-21T12:31:13.653 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:13 smithi189 sudo[29404]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-03-21T12:31:13.653 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:13 smithi189 sudo[29404]: pam_unix(sudo:session): session closed for user root 2024-03-21T12:31:14.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:13 smithi189 sudo[29407]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-03-21T12:31:14.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:13 smithi189 sudo[29407]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-03-21T12:31:14.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:13 smithi189 sudo[29407]: pam_unix(sudo:session): session closed for user root 2024-03-21T12:31:14.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:13 smithi189 ceph-mon[27080]: from='client.14294 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi022:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:31:14.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:13 smithi189 ceph-mon[27080]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-03-21T12:31:14.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:13 smithi189 ceph-mon[27080]: pgmap v74: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-03-21T12:31:14.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:13 smithi189 ceph-mon[27080]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-03-21T12:31:14.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:13 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mon metadata", "id": "smithi022"}]: dispatch 2024-03-21T12:31:14.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:13 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mon metadata", "id": "smithi189"}]: dispatch 2024-03-21T12:31:14.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:13 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mon metadata", "id": "smithi022"}]: dispatch 2024-03-21T12:31:14.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:13 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mon metadata", "id": "smithi189"}]: dispatch 2024-03-21T12:31:14.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:13 smithi022 ceph-mon[24777]: from='client.14294 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi022:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:31:14.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:13 smithi022 ceph-mon[24777]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-03-21T12:31:14.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:13 smithi022 ceph-mon[24777]: pgmap v74: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-03-21T12:31:14.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:13 smithi022 ceph-mon[24777]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-03-21T12:31:14.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:13 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mon metadata", "id": "smithi022"}]: dispatch 2024-03-21T12:31:14.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:13 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mon metadata", "id": "smithi189"}]: dispatch 2024-03-21T12:31:14.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:13 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mon metadata", "id": "smithi022"}]: dispatch 2024-03-21T12:31:14.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:13 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mon metadata", "id": "smithi189"}]: dispatch 2024-03-21T12:31:14.933 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:14 smithi022 ceph-mon[24777]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-03-21T12:31:14.933 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:14 smithi022 ceph-mon[24777]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-03-21T12:31:14.934 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:14 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:31:14.934 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:14 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:31:14.934 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:14 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-21T12:31:14.934 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:14 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-21T12:31:14.934 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:14 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-21T12:31:14.934 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:14 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:31:14.934 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:14 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:31:14.934 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:14 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:31:14.934 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:14 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:31:14.934 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:14 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:31:15.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:14 smithi189 ceph-mon[27080]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-03-21T12:31:15.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:14 smithi189 ceph-mon[27080]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-03-21T12:31:15.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:14 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:31:15.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:14 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:31:15.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:14 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-21T12:31:15.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:14 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-21T12:31:15.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:14 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-21T12:31:15.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:14 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:31:15.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:14 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:31:15.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:14 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:31:15.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:14 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:31:15.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:14 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:31:15.890 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:15 smithi022 ceph-mon[24777]: Adjusting osd_memory_target on smithi022 to 4289M 2024-03-21T12:31:15.890 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:15 smithi022 ceph-mon[24777]: pgmap v75: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2024-03-21T12:31:16.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:15 smithi189 ceph-mon[27080]: Adjusting osd_memory_target on smithi022 to 4289M 2024-03-21T12:31:16.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:15 smithi189 ceph-mon[27080]: pgmap v75: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2024-03-21T12:31:16.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:16 smithi022 ceph-mon[24777]: from='client.? 172.21.15.22:0/3787855712' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1fb58640-cff6-4041-af18-e3ab287610aa"}]: dispatch 2024-03-21T12:31:16.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:16 smithi022 ceph-mon[24777]: from='client.? 172.21.15.22:0/3787855712' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1fb58640-cff6-4041-af18-e3ab287610aa"}]': finished 2024-03-21T12:31:16.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:16 smithi022 ceph-mon[24777]: osdmap e24: 4 total, 3 up, 4 in 2024-03-21T12:31:16.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:16 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-21T12:31:16.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:16 smithi022 ceph-mon[24777]: from='client.? 172.21.15.22:0/4189669630' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-21T12:31:17.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:16 smithi189 ceph-mon[27080]: from='client.? 172.21.15.22:0/3787855712' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1fb58640-cff6-4041-af18-e3ab287610aa"}]: dispatch 2024-03-21T12:31:17.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:16 smithi189 ceph-mon[27080]: from='client.? 172.21.15.22:0/3787855712' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1fb58640-cff6-4041-af18-e3ab287610aa"}]': finished 2024-03-21T12:31:17.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:16 smithi189 ceph-mon[27080]: osdmap e24: 4 total, 3 up, 4 in 2024-03-21T12:31:17.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:16 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-21T12:31:17.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:16 smithi189 ceph-mon[27080]: from='client.? 172.21.15.22:0/4189669630' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-21T12:31:18.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:17 smithi022 ceph-mon[24777]: pgmap v77: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-03-21T12:31:18.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:17 smithi189 ceph-mon[27080]: pgmap v77: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-03-21T12:31:19.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:18 smithi022 ceph-mon[24777]: pgmap v78: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-03-21T12:31:19.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:18 smithi189 ceph-mon[27080]: pgmap v78: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-03-21T12:31:20.989 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:20 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:31:20.989 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:20 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:31:21.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:20 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:31:21.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:20 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:31:22.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:21 smithi189 ceph-mon[27080]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-03-21T12:31:22.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:21 smithi022 ceph-mon[24777]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-03-21T12:31:23.075 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:22 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-03-21T12:31:23.075 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:22 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:31:23.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:22 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-03-21T12:31:23.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:22 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:31:24.071 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:23 smithi022 ceph-mon[24777]: Deploying daemon osd.3 on smithi022 2024-03-21T12:31:24.071 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:23 smithi022 ceph-mon[24777]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-03-21T12:31:24.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:23 smithi189 ceph-mon[27080]: Deploying daemon osd.3 on smithi022 2024-03-21T12:31:24.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:23 smithi189 ceph-mon[27080]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-03-21T12:31:25.036 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:24 smithi022 ceph-mon[24777]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-03-21T12:31:25.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:24 smithi189 ceph-mon[27080]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-03-21T12:31:27.982 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:27 smithi022 ceph-mon[24777]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-03-21T12:31:27.983 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:27 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:31:27.983 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:27 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:31:27.983 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:27 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:31:27.983 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:27 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:31:28.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:27 smithi189 ceph-mon[27080]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-03-21T12:31:28.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:27 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:31:28.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:27 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:31:28.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:27 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:31:28.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:27 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:31:29.025 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:28 smithi022 ceph-mon[24777]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-03-21T12:31:29.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:28 smithi189 ceph-mon[27080]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-03-21T12:31:30.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:30 smithi189 ceph-mon[27080]: from='osd.3 [v2:172.21.15.22:6826/727407119,v1:172.21.15.22:6827/727407119]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-03-21T12:31:30.829 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:30 smithi022 ceph-mon[24777]: from='osd.3 [v2:172.21.15.22:6826/727407119,v1:172.21.15.22:6827/727407119]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-03-21T12:31:31.129 INFO:teuthology.orchestra.run.smithi022.stdout:Created osd(s) 3 on host 'smithi022' 2024-03-21T12:31:31.718 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:31 smithi022 ceph-mon[24777]: from='osd.3 [v2:172.21.15.22:6826/727407119,v1:172.21.15.22:6827/727407119]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-03-21T12:31:31.718 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:31 smithi022 ceph-mon[24777]: osdmap e25: 4 total, 3 up, 4 in 2024-03-21T12:31:31.718 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:31 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-21T12:31:31.718 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:31 smithi022 ceph-mon[24777]: from='osd.3 [v2:172.21.15.22:6826/727407119,v1:172.21.15.22:6827/727407119]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]: dispatch 2024-03-21T12:31:31.718 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:31 smithi022 ceph-mon[24777]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-03-21T12:31:31.718 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:31 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:31:31.742 DEBUG:teuthology.orchestra.run.smithi022:osd.3> sudo journalctl -f -n 0 -u ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715@osd.3.service 2024-03-21T12:31:31.744 INFO:tasks.cephadm:Deploying osd.4 on smithi189 with /dev/vg_nvme/lv_4... 2024-03-21T12:31:31.744 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-03-21T12:31:31.775 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:31 smithi189 ceph-mon[27080]: from='osd.3 [v2:172.21.15.22:6826/727407119,v1:172.21.15.22:6827/727407119]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-03-21T12:31:31.775 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:31 smithi189 ceph-mon[27080]: osdmap e25: 4 total, 3 up, 4 in 2024-03-21T12:31:31.776 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:31 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-21T12:31:31.776 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:31 smithi189 ceph-mon[27080]: from='osd.3 [v2:172.21.15.22:6826/727407119,v1:172.21.15.22:6827/727407119]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]: dispatch 2024-03-21T12:31:31.776 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:31 smithi189 ceph-mon[27080]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-03-21T12:31:31.776 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:31 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:31:32.612 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:32 smithi189 ceph-mon[27080]: from='osd.3 [v2:172.21.15.22:6826/727407119,v1:172.21.15.22:6827/727407119]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]': finished 2024-03-21T12:31:32.612 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:32 smithi189 ceph-mon[27080]: osdmap e26: 4 total, 3 up, 4 in 2024-03-21T12:31:32.612 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:32 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-21T12:31:32.612 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:32 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-21T12:31:32.788 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:32 smithi022 ceph-mon[24777]: from='osd.3 [v2:172.21.15.22:6826/727407119,v1:172.21.15.22:6827/727407119]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]': finished 2024-03-21T12:31:32.788 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:32 smithi022 ceph-mon[24777]: osdmap e26: 4 total, 3 up, 4 in 2024-03-21T12:31:32.788 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:32 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-21T12:31:32.788 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:32 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-21T12:31:33.654 INFO:teuthology.orchestra.run.smithi189.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-03-21T12:31:33.654 INFO:teuthology.orchestra.run.smithi189.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-03-21T12:31:33.654 INFO:teuthology.orchestra.run.smithi189.stderr: stderr: 10+0 records in 2024-03-21T12:31:33.654 INFO:teuthology.orchestra.run.smithi189.stderr:10+0 records out 2024-03-21T12:31:33.654 INFO:teuthology.orchestra.run.smithi189.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0112572 s, 931 MB/s 2024-03-21T12:31:33.654 INFO:teuthology.orchestra.run.smithi189.stderr:--> Zapping successful for: 2024-03-21T12:31:33.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:33 smithi189 ceph-mon[27080]: purged_snaps scrub starts 2024-03-21T12:31:33.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:33 smithi189 ceph-mon[27080]: purged_snaps scrub ok 2024-03-21T12:31:33.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:33 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-21T12:31:33.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:33 smithi189 ceph-mon[27080]: osd.3 [v2:172.21.15.22:6826/727407119,v1:172.21.15.22:6827/727407119] boot 2024-03-21T12:31:33.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:33 smithi189 ceph-mon[27080]: osdmap e27: 4 total, 4 up, 4 in 2024-03-21T12:31:33.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:33 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-21T12:31:33.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:33 smithi189 ceph-mon[27080]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-03-21T12:31:33.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:33 smithi022 ceph-mon[24777]: purged_snaps scrub starts 2024-03-21T12:31:33.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:33 smithi022 ceph-mon[24777]: purged_snaps scrub ok 2024-03-21T12:31:33.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:33 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-21T12:31:33.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:33 smithi022 ceph-mon[24777]: osd.3 [v2:172.21.15.22:6826/727407119,v1:172.21.15.22:6827/727407119] boot 2024-03-21T12:31:33.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:33 smithi022 ceph-mon[24777]: osdmap e27: 4 total, 4 up, 4 in 2024-03-21T12:31:33.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:33 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-21T12:31:33.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:33 smithi022 ceph-mon[24777]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-03-21T12:31:34.350 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -- ceph orch daemon add osd smithi189:vg_nvme/lv_4 2024-03-21T12:31:34.703 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:34 smithi189 ceph-mon[27080]: osdmap e28: 4 total, 4 up, 4 in 2024-03-21T12:31:34.704 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:34 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:31:34.757 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:34 smithi022 ceph-mon[24777]: osdmap e28: 4 total, 4 up, 4 in 2024-03-21T12:31:34.757 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:34 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:31:35.776 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:35 smithi022 ceph-mon[24777]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-21T12:31:35.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:35 smithi189 ceph-mon[27080]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-21T12:31:36.666 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:36 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-21T12:31:36.666 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:36 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-21T12:31:36.666 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:36 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:31:36.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:36 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-21T12:31:36.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:36 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-21T12:31:36.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:36 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:31:37.952 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:37 smithi189 ceph-mon[27080]: from='client.24145 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi189:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:31:37.953 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:37 smithi189 ceph-mon[27080]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-21T12:31:37.953 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:37 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:31:37.953 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:37 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-21T12:31:37.953 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:37 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-21T12:31:37.953 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:37 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-21T12:31:37.953 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:37 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-03-21T12:31:37.953 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:37 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:31:37.953 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:37 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:31:37.953 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:37 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:31:37.953 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:37 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:31:37.953 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:37 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:31:38.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:37 smithi022 ceph-mon[24777]: from='client.24145 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi189:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:31:38.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:37 smithi022 ceph-mon[24777]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-21T12:31:38.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:37 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:31:38.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:37 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-21T12:31:38.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:37 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-21T12:31:38.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:37 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-21T12:31:38.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:37 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-03-21T12:31:38.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:37 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:31:38.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:37 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:31:38.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:37 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:31:38.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:37 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:31:38.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:37 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:31:39.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:38 smithi022 ceph-mon[24777]: Detected new or changed devices on smithi022 2024-03-21T12:31:39.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:38 smithi022 ceph-mon[24777]: Adjusting osd_memory_target on smithi022 to 3217M 2024-03-21T12:31:39.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:38 smithi022 ceph-mon[24777]: from='client.? 172.21.15.189:0/2200076055' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "dc459155-aea6-4910-b1cc-566d824f3e91"}]: dispatch 2024-03-21T12:31:39.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:38 smithi022 ceph-mon[24777]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "dc459155-aea6-4910-b1cc-566d824f3e91"}]: dispatch 2024-03-21T12:31:39.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:38 smithi022 ceph-mon[24777]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "dc459155-aea6-4910-b1cc-566d824f3e91"}]': finished 2024-03-21T12:31:39.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:38 smithi022 ceph-mon[24777]: osdmap e29: 5 total, 4 up, 5 in 2024-03-21T12:31:39.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:38 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-21T12:31:39.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:38 smithi189 ceph-mon[27080]: Detected new or changed devices on smithi022 2024-03-21T12:31:39.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:38 smithi189 ceph-mon[27080]: Adjusting osd_memory_target on smithi022 to 3217M 2024-03-21T12:31:39.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:38 smithi189 ceph-mon[27080]: from='client.? 172.21.15.189:0/2200076055' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "dc459155-aea6-4910-b1cc-566d824f3e91"}]: dispatch 2024-03-21T12:31:39.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:38 smithi189 ceph-mon[27080]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "dc459155-aea6-4910-b1cc-566d824f3e91"}]: dispatch 2024-03-21T12:31:39.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:38 smithi189 ceph-mon[27080]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "dc459155-aea6-4910-b1cc-566d824f3e91"}]': finished 2024-03-21T12:31:39.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:38 smithi189 ceph-mon[27080]: osdmap e29: 5 total, 4 up, 5 in 2024-03-21T12:31:39.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:38 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-21T12:31:40.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:39 smithi022 ceph-mon[24777]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-21T12:31:40.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:39 smithi022 ceph-mon[24777]: from='client.? 172.21.15.189:0/2732771252' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-21T12:31:40.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:39 smithi189 ceph-mon[27080]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-21T12:31:40.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:39 smithi189 ceph-mon[27080]: from='client.? 172.21.15.189:0/2732771252' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-21T12:31:41.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:40 smithi022 ceph-mon[24777]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-21T12:31:41.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:40 smithi189 ceph-mon[27080]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-21T12:31:42.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:42 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:31:42.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:42 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:31:42.609 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:42 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:31:42.609 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:42 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:31:43.581 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:43 smithi189 ceph-mon[27080]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-21T12:31:43.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:43 smithi022 ceph-mon[24777]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-21T12:31:44.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:44 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-03-21T12:31:44.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:44 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:31:44.947 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:44 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-03-21T12:31:44.948 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:44 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:31:45.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:45 smithi022 ceph-mon[24777]: Deploying daemon osd.4 on smithi189 2024-03-21T12:31:45.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:45 smithi022 ceph-mon[24777]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-21T12:31:45.907 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:45 smithi189 ceph-mon[27080]: Deploying daemon osd.4 on smithi189 2024-03-21T12:31:45.908 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:45 smithi189 ceph-mon[27080]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-21T12:31:47.992 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:47 smithi189 ceph-mon[27080]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-21T12:31:48.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:47 smithi022 ceph-mon[24777]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-21T12:31:49.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:48 smithi189 ceph-mon[27080]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-21T12:31:49.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:48 smithi022 ceph-mon[24777]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-21T12:31:50.763 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:50 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:31:50.763 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:50 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:31:50.763 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:50 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:31:50.763 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:50 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:31:50.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:50 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:31:50.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:50 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:31:50.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:50 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:31:50.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:50 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:31:51.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:51 smithi189 ceph-mon[27080]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-21T12:31:51.746 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:51 smithi022 ceph-mon[24777]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-21T12:31:52.847 INFO:teuthology.orchestra.run.smithi189.stdout:Created osd(s) 4 on host 'smithi189' 2024-03-21T12:31:53.063 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:52 smithi189 ceph-mon[27080]: from='osd.4 [v2:172.21.15.189:6800/1650456785,v1:172.21.15.189:6801/1650456785]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-03-21T12:31:53.063 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:52 smithi189 ceph-mon[27080]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-03-21T12:31:53.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:52 smithi022 ceph-mon[24777]: from='osd.4 [v2:172.21.15.189:6800/1650456785,v1:172.21.15.189:6801/1650456785]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-03-21T12:31:53.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:52 smithi022 ceph-mon[24777]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-03-21T12:31:53.795 DEBUG:teuthology.orchestra.run.smithi189:osd.4> sudo journalctl -f -n 0 -u ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715@osd.4.service 2024-03-21T12:31:53.797 INFO:tasks.cephadm:Deploying osd.5 on smithi189 with /dev/vg_nvme/lv_3... 2024-03-21T12:31:53.798 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-03-21T12:31:54.035 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 21 12:31:53 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-4[31246]: 2024-03-21T12:31:53.826+0000 7fa49de14700 -1 osd.4 0 waiting for initial osdmap 2024-03-21T12:31:54.035 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 21 12:31:53 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-4[31246]: 2024-03-21T12:31:53.830+0000 7fa4967a7700 -1 osd.4 31 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-21T12:31:54.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:53 smithi189 ceph-mon[27080]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-03-21T12:31:54.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:53 smithi189 ceph-mon[27080]: osdmap e30: 5 total, 4 up, 5 in 2024-03-21T12:31:54.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:53 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-21T12:31:54.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:53 smithi189 ceph-mon[27080]: from='osd.4 [v2:172.21.15.189:6800/1650456785,v1:172.21.15.189:6801/1650456785]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-03-21T12:31:54.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:53 smithi189 ceph-mon[27080]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-03-21T12:31:54.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:53 smithi189 ceph-mon[27080]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-21T12:31:54.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:53 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:31:54.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:53 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:31:54.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:53 smithi022 ceph-mon[24777]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-03-21T12:31:54.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:53 smithi022 ceph-mon[24777]: osdmap e30: 5 total, 4 up, 5 in 2024-03-21T12:31:54.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:53 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-21T12:31:54.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:53 smithi022 ceph-mon[24777]: from='osd.4 [v2:172.21.15.189:6800/1650456785,v1:172.21.15.189:6801/1650456785]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-03-21T12:31:54.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:53 smithi022 ceph-mon[24777]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-03-21T12:31:54.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:53 smithi022 ceph-mon[24777]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-21T12:31:54.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:53 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:31:54.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:53 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:31:55.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:54 smithi022 ceph-mon[24777]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]': finished 2024-03-21T12:31:55.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:54 smithi022 ceph-mon[24777]: osdmap e31: 5 total, 4 up, 5 in 2024-03-21T12:31:55.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:54 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-21T12:31:55.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:54 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-21T12:31:55.169 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:54 smithi189 ceph-mon[27080]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]': finished 2024-03-21T12:31:55.169 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:54 smithi189 ceph-mon[27080]: osdmap e31: 5 total, 4 up, 5 in 2024-03-21T12:31:55.169 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:54 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-21T12:31:55.169 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:54 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-21T12:31:56.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:55 smithi022 ceph-mon[24777]: purged_snaps scrub starts 2024-03-21T12:31:56.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:55 smithi022 ceph-mon[24777]: purged_snaps scrub ok 2024-03-21T12:31:56.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:55 smithi022 ceph-mon[24777]: osd.4 [v2:172.21.15.189:6800/1650456785,v1:172.21.15.189:6801/1650456785] boot 2024-03-21T12:31:56.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:55 smithi022 ceph-mon[24777]: osdmap e32: 5 total, 5 up, 5 in 2024-03-21T12:31:56.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:55 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-21T12:31:56.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:55 smithi022 ceph-mon[24777]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-21T12:31:56.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:55 smithi022 ceph-mon[24777]: osdmap e33: 5 total, 5 up, 5 in 2024-03-21T12:31:56.095 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:55 smithi189 ceph-mon[27080]: purged_snaps scrub starts 2024-03-21T12:31:56.096 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:55 smithi189 ceph-mon[27080]: purged_snaps scrub ok 2024-03-21T12:31:56.096 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:55 smithi189 ceph-mon[27080]: osd.4 [v2:172.21.15.189:6800/1650456785,v1:172.21.15.189:6801/1650456785] boot 2024-03-21T12:31:56.096 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:55 smithi189 ceph-mon[27080]: osdmap e32: 5 total, 5 up, 5 in 2024-03-21T12:31:56.096 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:55 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-21T12:31:56.096 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:55 smithi189 ceph-mon[27080]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-21T12:31:56.096 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:55 smithi189 ceph-mon[27080]: osdmap e33: 5 total, 5 up, 5 in 2024-03-21T12:31:56.390 INFO:teuthology.orchestra.run.smithi189.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-03-21T12:31:56.390 INFO:teuthology.orchestra.run.smithi189.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-03-21T12:31:56.390 INFO:teuthology.orchestra.run.smithi189.stderr: stderr: 10+0 records in 2024-03-21T12:31:56.390 INFO:teuthology.orchestra.run.smithi189.stderr:10+0 records out 2024-03-21T12:31:56.390 INFO:teuthology.orchestra.run.smithi189.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0122787 s, 854 MB/s 2024-03-21T12:31:56.390 INFO:teuthology.orchestra.run.smithi189.stderr: stderr: 2024-03-21T12:31:56.390 INFO:teuthology.orchestra.run.smithi189.stderr:--> Zapping successful for: 2024-03-21T12:31:57.033 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -- ceph orch daemon add osd smithi189:vg_nvme/lv_3 2024-03-21T12:31:58.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:57 smithi022 ceph-mon[24777]: osdmap e34: 5 total, 5 up, 5 in 2024-03-21T12:31:58.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:57 smithi022 ceph-mon[24777]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-03-21T12:31:58.223 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:57 smithi189 ceph-mon[27080]: osdmap e34: 5 total, 5 up, 5 in 2024-03-21T12:31:58.223 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:57 smithi189 ceph-mon[27080]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-03-21T12:31:59.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:58 smithi022 ceph-mon[24777]: osdmap e35: 5 total, 5 up, 5 in 2024-03-21T12:31:59.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:58 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:31:59.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:58 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-21T12:31:59.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:58 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:31:59.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:58 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:31:59.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:58 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:31:59.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:58 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:31:59.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:58 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:31:59.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:58 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:31:59.165 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:58 smithi189 ceph-mon[27080]: osdmap e35: 5 total, 5 up, 5 in 2024-03-21T12:31:59.165 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:58 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:31:59.166 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:58 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-21T12:31:59.166 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:58 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:31:59.166 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:58 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:31:59.166 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:58 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:31:59.166 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:58 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:31:59.166 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:58 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:31:59.166 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:58 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:32:00.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:59 smithi022 ceph-mon[24777]: Detected new or changed devices on smithi189 2024-03-21T12:32:00.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:59 smithi022 ceph-mon[24777]: Adjusting osd_memory_target on smithi189 to 15936M 2024-03-21T12:32:00.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:59 smithi022 ceph-mon[24777]: osdmap e36: 5 total, 5 up, 5 in 2024-03-21T12:32:00.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:59 smithi022 ceph-mon[24777]: pgmap v110: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-03-21T12:32:00.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:59 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-21T12:32:00.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:59 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-21T12:32:00.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:31:59 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:32:00.194 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:59 smithi189 ceph-mon[27080]: Detected new or changed devices on smithi189 2024-03-21T12:32:00.194 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:59 smithi189 ceph-mon[27080]: Adjusting osd_memory_target on smithi189 to 15936M 2024-03-21T12:32:00.195 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:59 smithi189 ceph-mon[27080]: osdmap e36: 5 total, 5 up, 5 in 2024-03-21T12:32:00.195 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:59 smithi189 ceph-mon[27080]: pgmap v110: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-03-21T12:32:00.195 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:59 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-21T12:32:00.195 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:59 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-21T12:32:00.195 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:31:59 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:32:01.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:00 smithi022 ceph-mon[24777]: from='client.24163 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi189:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:32:01.216 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:00 smithi189 ceph-mon[27080]: from='client.24163 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi189:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:32:02.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:01 smithi022 ceph-mon[24777]: pgmap v111: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 103 KiB/s, 0 objects/s recovering 2024-03-21T12:32:02.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:01 smithi189 ceph-mon[27080]: pgmap v111: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 103 KiB/s, 0 objects/s recovering 2024-03-21T12:32:03.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:02 smithi022 ceph-mon[24777]: from='client.? 172.21.15.189:0/1534127131' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3712ad88-b7d4-4b7a-bc34-0ec6bfa92e67"}]: dispatch 2024-03-21T12:32:03.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:02 smithi022 ceph-mon[24777]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3712ad88-b7d4-4b7a-bc34-0ec6bfa92e67"}]: dispatch 2024-03-21T12:32:03.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:02 smithi022 ceph-mon[24777]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3712ad88-b7d4-4b7a-bc34-0ec6bfa92e67"}]': finished 2024-03-21T12:32:03.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:02 smithi022 ceph-mon[24777]: osdmap e37: 6 total, 5 up, 6 in 2024-03-21T12:32:03.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:02 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-21T12:32:03.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:02 smithi022 ceph-mon[24777]: pgmap v113: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-03-21T12:32:03.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:02 smithi022 ceph-mon[24777]: from='client.? 172.21.15.189:0/538975020' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-21T12:32:03.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:02 smithi189 ceph-mon[27080]: from='client.? 172.21.15.189:0/1534127131' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3712ad88-b7d4-4b7a-bc34-0ec6bfa92e67"}]: dispatch 2024-03-21T12:32:03.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:02 smithi189 ceph-mon[27080]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3712ad88-b7d4-4b7a-bc34-0ec6bfa92e67"}]: dispatch 2024-03-21T12:32:03.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:02 smithi189 ceph-mon[27080]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3712ad88-b7d4-4b7a-bc34-0ec6bfa92e67"}]': finished 2024-03-21T12:32:03.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:02 smithi189 ceph-mon[27080]: osdmap e37: 6 total, 5 up, 6 in 2024-03-21T12:32:03.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:02 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-21T12:32:03.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:02 smithi189 ceph-mon[27080]: pgmap v113: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-03-21T12:32:03.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:02 smithi189 ceph-mon[27080]: from='client.? 172.21.15.189:0/538975020' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-21T12:32:05.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:04 smithi022 ceph-mon[24777]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 74 KiB/s, 0 objects/s recovering 2024-03-21T12:32:05.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:04 smithi189 ceph-mon[27080]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 74 KiB/s, 0 objects/s recovering 2024-03-21T12:32:08.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:07 smithi189 ceph-mon[27080]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-03-21T12:32:08.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:07 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi022.fektzs/mirror_snapshot_schedule"}]: dispatch 2024-03-21T12:32:08.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:07 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi022.fektzs/trash_purge_schedule"}]: dispatch 2024-03-21T12:32:08.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:07 smithi022 ceph-mon[24777]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-03-21T12:32:08.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:07 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi022.fektzs/mirror_snapshot_schedule"}]: dispatch 2024-03-21T12:32:08.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:07 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi022.fektzs/trash_purge_schedule"}]: dispatch 2024-03-21T12:32:08.889 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:08 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-03-21T12:32:08.889 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:08 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:32:09.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:08 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-03-21T12:32:09.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:08 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:32:10.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:09 smithi189 ceph-mon[27080]: Deploying daemon osd.5 on smithi189 2024-03-21T12:32:10.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:09 smithi189 ceph-mon[27080]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-03-21T12:32:10.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:09 smithi022 ceph-mon[24777]: Deploying daemon osd.5 on smithi189 2024-03-21T12:32:10.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:09 smithi022 ceph-mon[24777]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-03-21T12:32:11.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:10 smithi189 ceph-mon[27080]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-03-21T12:32:11.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:10 smithi022 ceph-mon[24777]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-03-21T12:32:13.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:12 smithi189 ceph-mon[27080]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-03-21T12:32:13.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:12 smithi022 ceph-mon[24777]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-03-21T12:32:14.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:13 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:32:14.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:13 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:32:14.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:13 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:32:14.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:13 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:32:14.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:13 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:32:14.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:13 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:32:14.345 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:13 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:32:14.345 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:13 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:32:15.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:14 smithi022 ceph-mon[24777]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-03-21T12:32:15.280 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:14 smithi189 ceph-mon[27080]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-03-21T12:32:16.775 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:16 smithi189 ceph-mon[27080]: from='osd.5 [v2:172.21.15.189:6808/2147670707,v1:172.21.15.189:6809/2147670707]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-03-21T12:32:16.775 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:16 smithi189 ceph-mon[27080]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-03-21T12:32:16.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:16 smithi022 ceph-mon[24777]: from='osd.5 [v2:172.21.15.189:6808/2147670707,v1:172.21.15.189:6809/2147670707]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-03-21T12:32:16.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:16 smithi022 ceph-mon[24777]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-03-21T12:32:17.082 INFO:teuthology.orchestra.run.smithi189.stdout:Created osd(s) 5 on host 'smithi189' 2024-03-21T12:32:17.744 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:17 smithi189 ceph-mon[27080]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-03-21T12:32:17.744 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:17 smithi189 ceph-mon[27080]: osdmap e38: 6 total, 5 up, 6 in 2024-03-21T12:32:17.744 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:17 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-21T12:32:17.744 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:17 smithi189 ceph-mon[27080]: from='osd.5 [v2:172.21.15.189:6808/2147670707,v1:172.21.15.189:6809/2147670707]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-03-21T12:32:17.744 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:17 smithi189 ceph-mon[27080]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-03-21T12:32:17.745 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:17 smithi189 ceph-mon[27080]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-03-21T12:32:17.745 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:17 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:32:17.745 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:17 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:32:17.772 DEBUG:teuthology.orchestra.run.smithi189:osd.5> sudo journalctl -f -n 0 -u ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715@osd.5.service 2024-03-21T12:32:17.774 INFO:tasks.cephadm:Deploying osd.6 on smithi189 with /dev/vg_nvme/lv_2... 2024-03-21T12:32:17.774 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-03-21T12:32:17.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:17 smithi022 ceph-mon[24777]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-03-21T12:32:17.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:17 smithi022 ceph-mon[24777]: osdmap e38: 6 total, 5 up, 6 in 2024-03-21T12:32:17.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:17 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-21T12:32:17.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:17 smithi022 ceph-mon[24777]: from='osd.5 [v2:172.21.15.189:6808/2147670707,v1:172.21.15.189:6809/2147670707]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-03-21T12:32:17.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:17 smithi022 ceph-mon[24777]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-03-21T12:32:17.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:17 smithi022 ceph-mon[24777]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-03-21T12:32:17.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:17 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:32:17.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:17 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:32:18.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:18 smithi189 ceph-mon[27080]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]': finished 2024-03-21T12:32:18.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:18 smithi189 ceph-mon[27080]: osdmap e39: 6 total, 5 up, 6 in 2024-03-21T12:32:18.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:18 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-21T12:32:18.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:18 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-21T12:32:18.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:18 smithi022 ceph-mon[24777]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]': finished 2024-03-21T12:32:18.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:18 smithi022 ceph-mon[24777]: osdmap e39: 6 total, 5 up, 6 in 2024-03-21T12:32:18.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:18 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-21T12:32:18.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:18 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-21T12:32:19.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:19 smithi189 ceph-mon[27080]: purged_snaps scrub starts 2024-03-21T12:32:19.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:19 smithi189 ceph-mon[27080]: purged_snaps scrub ok 2024-03-21T12:32:19.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:19 smithi189 ceph-mon[27080]: osd.5 [v2:172.21.15.189:6808/2147670707,v1:172.21.15.189:6809/2147670707] boot 2024-03-21T12:32:19.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:19 smithi189 ceph-mon[27080]: osdmap e40: 6 total, 6 up, 6 in 2024-03-21T12:32:19.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:19 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-21T12:32:19.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:19 smithi189 ceph-mon[27080]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-03-21T12:32:19.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:19 smithi022 ceph-mon[24777]: purged_snaps scrub starts 2024-03-21T12:32:19.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:19 smithi022 ceph-mon[24777]: purged_snaps scrub ok 2024-03-21T12:32:19.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:19 smithi022 ceph-mon[24777]: osd.5 [v2:172.21.15.189:6808/2147670707,v1:172.21.15.189:6809/2147670707] boot 2024-03-21T12:32:19.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:19 smithi022 ceph-mon[24777]: osdmap e40: 6 total, 6 up, 6 in 2024-03-21T12:32:19.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:19 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-21T12:32:19.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:19 smithi022 ceph-mon[24777]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-03-21T12:32:20.756 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:20 smithi189 ceph-mon[27080]: osdmap e41: 6 total, 6 up, 6 in 2024-03-21T12:32:20.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:20 smithi022 ceph-mon[24777]: osdmap e41: 6 total, 6 up, 6 in 2024-03-21T12:32:21.139 INFO:teuthology.orchestra.run.smithi189.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-03-21T12:32:21.139 INFO:teuthology.orchestra.run.smithi189.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-03-21T12:32:21.139 INFO:teuthology.orchestra.run.smithi189.stderr: stderr: 10+0 records in 2024-03-21T12:32:21.139 INFO:teuthology.orchestra.run.smithi189.stderr:10+0 records out 2024-03-21T12:32:21.140 INFO:teuthology.orchestra.run.smithi189.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0122653 s, 855 MB/s 2024-03-21T12:32:21.140 INFO:teuthology.orchestra.run.smithi189.stderr:--> Zapping successful for: 2024-03-21T12:32:21.696 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:21 smithi189 ceph-mon[27080]: osdmap e42: 6 total, 6 up, 6 in 2024-03-21T12:32:21.696 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:21 smithi189 ceph-mon[27080]: pgmap v127: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail; 2/6 objects degraded (33.333%) 2024-03-21T12:32:21.846 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:21 smithi022 ceph-mon[24777]: osdmap e42: 6 total, 6 up, 6 in 2024-03-21T12:32:21.846 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:21 smithi022 ceph-mon[24777]: pgmap v127: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail; 2/6 objects degraded (33.333%) 2024-03-21T12:32:22.265 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -- ceph orch daemon add osd smithi189:vg_nvme/lv_2 2024-03-21T12:32:22.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:22 smithi022 ceph-mon[24777]: Health check failed: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded (PG_DEGRADED) 2024-03-21T12:32:22.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:22 smithi022 ceph-mon[24777]: osdmap e43: 6 total, 6 up, 6 in 2024-03-21T12:32:22.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:22 smithi022 ceph-mon[24777]: Detected new or changed devices on smithi189 2024-03-21T12:32:22.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:22 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:32:22.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:22 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-21T12:32:22.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:22 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-21T12:32:22.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:22 smithi022 ceph-mon[24777]: Adjusting osd_memory_target on smithi189 to 7968M 2024-03-21T12:32:22.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:22 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:32:22.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:22 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:32:22.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:22 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:32:22.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:22 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:32:22.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:22 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:32:22.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:22 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:32:22.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:22 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:32:22.859 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:22 smithi189 ceph-mon[27080]: Health check failed: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded (PG_DEGRADED) 2024-03-21T12:32:22.859 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:22 smithi189 ceph-mon[27080]: osdmap e43: 6 total, 6 up, 6 in 2024-03-21T12:32:22.859 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:22 smithi189 ceph-mon[27080]: Detected new or changed devices on smithi189 2024-03-21T12:32:22.859 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:22 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:32:22.859 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:22 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-21T12:32:22.859 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:22 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-21T12:32:22.859 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:22 smithi189 ceph-mon[27080]: Adjusting osd_memory_target on smithi189 to 7968M 2024-03-21T12:32:22.859 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:22 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:32:22.860 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:22 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:32:22.860 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:22 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:32:22.860 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:22 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:32:22.860 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:22 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:32:22.860 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:22 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:32:22.860 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:22 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:32:23.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:23 smithi022 ceph-mon[24777]: osdmap e44: 6 total, 6 up, 6 in 2024-03-21T12:32:23.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:23 smithi022 ceph-mon[24777]: pgmap v130: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail; 2/6 objects degraded (33.333%) 2024-03-21T12:32:23.882 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:23 smithi189 ceph-mon[27080]: osdmap e44: 6 total, 6 up, 6 in 2024-03-21T12:32:23.882 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:23 smithi189 ceph-mon[27080]: pgmap v130: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail; 2/6 objects degraded (33.333%) 2024-03-21T12:32:24.818 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:24 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-21T12:32:24.818 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:24 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-21T12:32:24.818 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:24 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:32:24.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:24 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-21T12:32:24.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:24 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-21T12:32:24.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:24 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:32:25.639 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:25 smithi189 ceph-mon[27080]: from='client.24181 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi189:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:32:25.640 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:25 smithi189 ceph-mon[27080]: pgmap v131: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-03-21T12:32:25.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:25 smithi022 ceph-mon[24777]: from='client.24181 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi189:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:32:25.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:25 smithi022 ceph-mon[24777]: pgmap v131: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-03-21T12:32:26.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:26 smithi022 ceph-mon[24777]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded) 2024-03-21T12:32:26.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:26 smithi022 ceph-mon[24777]: Cluster is now healthy 2024-03-21T12:32:26.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:26 smithi022 ceph-mon[24777]: from='client.? 172.21.15.189:0/740399557' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fda195d0-e8ef-47eb-9424-6913e213c145"}]: dispatch 2024-03-21T12:32:26.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:26 smithi022 ceph-mon[24777]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fda195d0-e8ef-47eb-9424-6913e213c145"}]: dispatch 2024-03-21T12:32:26.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:26 smithi022 ceph-mon[24777]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "fda195d0-e8ef-47eb-9424-6913e213c145"}]': finished 2024-03-21T12:32:26.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:26 smithi022 ceph-mon[24777]: osdmap e45: 7 total, 6 up, 7 in 2024-03-21T12:32:26.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:26 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-21T12:32:27.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:26 smithi189 ceph-mon[27080]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded) 2024-03-21T12:32:27.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:26 smithi189 ceph-mon[27080]: Cluster is now healthy 2024-03-21T12:32:27.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:26 smithi189 ceph-mon[27080]: from='client.? 172.21.15.189:0/740399557' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fda195d0-e8ef-47eb-9424-6913e213c145"}]: dispatch 2024-03-21T12:32:27.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:26 smithi189 ceph-mon[27080]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fda195d0-e8ef-47eb-9424-6913e213c145"}]: dispatch 2024-03-21T12:32:27.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:26 smithi189 ceph-mon[27080]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "fda195d0-e8ef-47eb-9424-6913e213c145"}]': finished 2024-03-21T12:32:27.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:26 smithi189 ceph-mon[27080]: osdmap e45: 7 total, 6 up, 7 in 2024-03-21T12:32:27.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:26 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-21T12:32:28.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:27 smithi022 ceph-mon[24777]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-03-21T12:32:28.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:27 smithi022 ceph-mon[24777]: from='client.? 172.21.15.189:0/1580037603' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-21T12:32:28.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:27 smithi189 ceph-mon[27080]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-03-21T12:32:28.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:27 smithi189 ceph-mon[27080]: from='client.? 172.21.15.189:0/1580037603' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-21T12:32:29.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:28 smithi189 ceph-mon[27080]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 71 KiB/s, 0 objects/s recovering 2024-03-21T12:32:29.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:28 smithi022 ceph-mon[24777]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 71 KiB/s, 0 objects/s recovering 2024-03-21T12:32:31.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:30 smithi189 ceph-mon[27080]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 62 KiB/s, 0 objects/s recovering 2024-03-21T12:32:31.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:30 smithi022 ceph-mon[24777]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 62 KiB/s, 0 objects/s recovering 2024-03-21T12:32:33.278 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:32 smithi189 ceph-mon[27080]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-03-21T12:32:33.279 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:32 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-03-21T12:32:33.279 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:32 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:32:33.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:32 smithi022 ceph-mon[24777]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-03-21T12:32:33.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:32 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-03-21T12:32:33.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:32 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:32:34.191 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:33 smithi189 ceph-mon[27080]: Deploying daemon osd.6 on smithi189 2024-03-21T12:32:34.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:33 smithi022 ceph-mon[24777]: Deploying daemon osd.6 on smithi189 2024-03-21T12:32:35.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:34 smithi022 ceph-mon[24777]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-03-21T12:32:35.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:34 smithi189 ceph-mon[27080]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-03-21T12:32:38.000 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:37 smithi189 ceph-mon[27080]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-03-21T12:32:38.000 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:37 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:32:38.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:37 smithi022 ceph-mon[24777]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-03-21T12:32:38.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:37 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:32:39.218 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:38 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:32:39.218 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:38 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:32:39.218 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:38 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:32:39.218 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:38 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:32:39.218 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:38 smithi189 ceph-mon[27080]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-03-21T12:32:39.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:38 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:32:39.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:38 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:32:39.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:38 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:32:39.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:38 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:32:39.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:38 smithi022 ceph-mon[24777]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-03-21T12:32:41.202 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:40 smithi189 ceph-mon[27080]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-03-21T12:32:41.233 INFO:teuthology.orchestra.run.smithi189.stdout:Created osd(s) 6 on host 'smithi189' 2024-03-21T12:32:41.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:40 smithi022 ceph-mon[24777]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-03-21T12:32:41.829 DEBUG:teuthology.orchestra.run.smithi189:osd.6> sudo journalctl -f -n 0 -u ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715@osd.6.service 2024-03-21T12:32:41.832 INFO:tasks.cephadm:Deploying osd.7 on smithi189 with /dev/vg_nvme/lv_1... 2024-03-21T12:32:41.832 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-03-21T12:32:42.123 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:41 smithi189 ceph-mon[27080]: from='osd.6 [v2:172.21.15.189:6816/1085921107,v1:172.21.15.189:6817/1085921107]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-03-21T12:32:42.124 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:41 smithi189 ceph-mon[27080]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-03-21T12:32:42.124 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:41 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:32:42.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:41 smithi022 ceph-mon[24777]: from='osd.6 [v2:172.21.15.189:6816/1085921107,v1:172.21.15.189:6817/1085921107]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-03-21T12:32:42.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:41 smithi022 ceph-mon[24777]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-03-21T12:32:42.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:41 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:32:43.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:42 smithi189 ceph-mon[27080]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-03-21T12:32:43.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:42 smithi189 ceph-mon[27080]: osdmap e46: 7 total, 6 up, 7 in 2024-03-21T12:32:43.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:42 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-21T12:32:43.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:42 smithi189 ceph-mon[27080]: from='osd.6 [v2:172.21.15.189:6816/1085921107,v1:172.21.15.189:6817/1085921107]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-03-21T12:32:43.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:42 smithi189 ceph-mon[27080]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-03-21T12:32:43.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:42 smithi189 ceph-mon[27080]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-03-21T12:32:43.036 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:32:42 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6[37273]: 2024-03-21T12:32:42.919+0000 7f70a528e700 -1 osd.6 0 waiting for initial osdmap 2024-03-21T12:32:43.036 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:32:42 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6[37273]: 2024-03-21T12:32:42.927+0000 7f709fc25700 -1 osd.6 47 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-21T12:32:43.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:42 smithi022 ceph-mon[24777]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-03-21T12:32:43.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:42 smithi022 ceph-mon[24777]: osdmap e46: 7 total, 6 up, 7 in 2024-03-21T12:32:43.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:42 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-21T12:32:43.345 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:42 smithi022 ceph-mon[24777]: from='osd.6 [v2:172.21.15.189:6816/1085921107,v1:172.21.15.189:6817/1085921107]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-03-21T12:32:43.345 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:42 smithi022 ceph-mon[24777]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-03-21T12:32:43.345 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:42 smithi022 ceph-mon[24777]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-03-21T12:32:44.110 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:43 smithi189 ceph-mon[27080]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]': finished 2024-03-21T12:32:44.110 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:43 smithi189 ceph-mon[27080]: osdmap e47: 7 total, 6 up, 7 in 2024-03-21T12:32:44.110 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:43 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-21T12:32:44.110 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:43 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-21T12:32:44.110 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:43 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:32:44.237 INFO:teuthology.orchestra.run.smithi189.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-03-21T12:32:44.237 INFO:teuthology.orchestra.run.smithi189.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-03-21T12:32:44.237 INFO:teuthology.orchestra.run.smithi189.stderr: stderr: 10+0 records in 2024-03-21T12:32:44.237 INFO:teuthology.orchestra.run.smithi189.stderr:10+0 records out 2024-03-21T12:32:44.237 INFO:teuthology.orchestra.run.smithi189.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0126543 s, 829 MB/s 2024-03-21T12:32:44.237 INFO:teuthology.orchestra.run.smithi189.stderr:--> Zapping successful for: 2024-03-21T12:32:44.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:43 smithi022 ceph-mon[24777]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]': finished 2024-03-21T12:32:44.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:43 smithi022 ceph-mon[24777]: osdmap e47: 7 total, 6 up, 7 in 2024-03-21T12:32:44.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:43 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-21T12:32:44.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:43 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-21T12:32:44.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:43 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:32:45.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:44 smithi189 ceph-mon[27080]: purged_snaps scrub starts 2024-03-21T12:32:45.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:44 smithi189 ceph-mon[27080]: purged_snaps scrub ok 2024-03-21T12:32:45.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:44 smithi189 ceph-mon[27080]: osd.6 [v2:172.21.15.189:6816/1085921107,v1:172.21.15.189:6817/1085921107] boot 2024-03-21T12:32:45.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:44 smithi189 ceph-mon[27080]: osdmap e48: 7 total, 7 up, 7 in 2024-03-21T12:32:45.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:44 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-21T12:32:45.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:44 smithi189 ceph-mon[27080]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-03-21T12:32:45.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:44 smithi022 ceph-mon[24777]: purged_snaps scrub starts 2024-03-21T12:32:45.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:44 smithi022 ceph-mon[24777]: purged_snaps scrub ok 2024-03-21T12:32:45.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:44 smithi022 ceph-mon[24777]: osd.6 [v2:172.21.15.189:6816/1085921107,v1:172.21.15.189:6817/1085921107] boot 2024-03-21T12:32:45.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:44 smithi022 ceph-mon[24777]: osdmap e48: 7 total, 7 up, 7 in 2024-03-21T12:32:45.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:44 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-21T12:32:45.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:44 smithi022 ceph-mon[24777]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-03-21T12:32:45.350 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -- ceph orch daemon add osd smithi189:vg_nvme/lv_1 2024-03-21T12:32:45.990 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:45 smithi189 ceph-mon[27080]: osdmap e49: 7 total, 7 up, 7 in 2024-03-21T12:32:46.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:45 smithi022 ceph-mon[24777]: osdmap e49: 7 total, 7 up, 7 in 2024-03-21T12:32:47.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:46 smithi189 ceph-mon[27080]: osdmap e50: 7 total, 7 up, 7 in 2024-03-21T12:32:47.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:46 smithi189 ceph-mon[27080]: osdmap e51: 7 total, 7 up, 7 in 2024-03-21T12:32:47.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:46 smithi189 ceph-mon[27080]: pgmap v149: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-03-21T12:32:47.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:46 smithi022 ceph-mon[24777]: osdmap e50: 7 total, 7 up, 7 in 2024-03-21T12:32:47.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:46 smithi022 ceph-mon[24777]: osdmap e51: 7 total, 7 up, 7 in 2024-03-21T12:32:47.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:46 smithi022 ceph-mon[24777]: pgmap v149: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-03-21T12:32:49.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:48 smithi189 ceph-mon[27080]: Detected new or changed devices on smithi189 2024-03-21T12:32:49.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:48 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:32:49.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:48 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-21T12:32:49.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:48 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-21T12:32:49.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:48 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-21T12:32:49.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:48 smithi189 ceph-mon[27080]: Adjusting osd_memory_target on smithi189 to 5312M 2024-03-21T12:32:49.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:48 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:32:49.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:48 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:32:49.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:48 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:32:49.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:48 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:32:49.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:48 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:32:49.037 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:48 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:32:49.037 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:48 smithi189 ceph-mon[27080]: osdmap e52: 7 total, 7 up, 7 in 2024-03-21T12:32:49.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:48 smithi022 ceph-mon[24777]: Detected new or changed devices on smithi189 2024-03-21T12:32:49.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:48 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:32:49.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:48 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-21T12:32:49.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:48 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-21T12:32:49.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:48 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-21T12:32:49.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:48 smithi022 ceph-mon[24777]: Adjusting osd_memory_target on smithi189 to 5312M 2024-03-21T12:32:49.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:48 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:32:49.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:48 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:32:49.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:48 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:32:49.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:48 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:32:49.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:48 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:32:49.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:48 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:32:49.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:48 smithi022 ceph-mon[24777]: osdmap e52: 7 total, 7 up, 7 in 2024-03-21T12:32:50.012 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:49 smithi189 ceph-mon[27080]: from='client.24199 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi189:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:32:50.012 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:49 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-21T12:32:50.012 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:49 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-21T12:32:50.012 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:49 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:32:50.012 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:49 smithi189 ceph-mon[27080]: pgmap v151: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-03-21T12:32:50.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:49 smithi022 ceph-mon[24777]: from='client.24199 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi189:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:32:50.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:49 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-21T12:32:50.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:49 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-21T12:32:50.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:49 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:32:50.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:49 smithi022 ceph-mon[24777]: pgmap v151: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-03-21T12:32:51.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:50 smithi189 ceph-mon[27080]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 87 KiB/s, 0 objects/s recovering 2024-03-21T12:32:51.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:50 smithi189 ceph-mon[27080]: from='client.? 172.21.15.189:0/4167926816' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "22d112d7-aca4-4a57-bd2a-c5890251bee2"}]: dispatch 2024-03-21T12:32:51.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:50 smithi189 ceph-mon[27080]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "22d112d7-aca4-4a57-bd2a-c5890251bee2"}]: dispatch 2024-03-21T12:32:51.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:50 smithi189 ceph-mon[27080]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "22d112d7-aca4-4a57-bd2a-c5890251bee2"}]': finished 2024-03-21T12:32:51.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:50 smithi189 ceph-mon[27080]: osdmap e53: 8 total, 7 up, 8 in 2024-03-21T12:32:51.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:50 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-21T12:32:51.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:50 smithi022 ceph-mon[24777]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 87 KiB/s, 0 objects/s recovering 2024-03-21T12:32:51.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:50 smithi022 ceph-mon[24777]: from='client.? 172.21.15.189:0/4167926816' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "22d112d7-aca4-4a57-bd2a-c5890251bee2"}]: dispatch 2024-03-21T12:32:51.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:50 smithi022 ceph-mon[24777]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "22d112d7-aca4-4a57-bd2a-c5890251bee2"}]: dispatch 2024-03-21T12:32:51.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:50 smithi022 ceph-mon[24777]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "22d112d7-aca4-4a57-bd2a-c5890251bee2"}]': finished 2024-03-21T12:32:51.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:50 smithi022 ceph-mon[24777]: osdmap e53: 8 total, 7 up, 8 in 2024-03-21T12:32:51.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:50 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-21T12:32:52.245 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:51 smithi022 ceph-mon[24777]: from='client.? 172.21.15.189:0/2485608004' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-21T12:32:52.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:51 smithi189 ceph-mon[27080]: from='client.? 172.21.15.189:0/2485608004' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-21T12:32:53.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:52 smithi189 ceph-mon[27080]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-03-21T12:32:53.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:52 smithi022 ceph-mon[24777]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-03-21T12:32:55.014 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:54 smithi189 ceph-mon[27080]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-03-21T12:32:55.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:54 smithi022 ceph-mon[24777]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-03-21T12:32:58.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:57 smithi022 ceph-mon[24777]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 57 KiB/s, 0 objects/s recovering 2024-03-21T12:32:58.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:57 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-03-21T12:32:58.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:57 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:32:58.155 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:57 smithi189 ceph-mon[27080]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 57 KiB/s, 0 objects/s recovering 2024-03-21T12:32:58.155 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:57 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-03-21T12:32:58.155 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:57 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:32:59.033 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:58 smithi189 ceph-mon[27080]: Deploying daemon osd.7 on smithi189 2024-03-21T12:32:59.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:58 smithi022 ceph-mon[24777]: Deploying daemon osd.7 on smithi189 2024-03-21T12:33:00.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:32:59 smithi189 ceph-mon[27080]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-03-21T12:33:00.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:32:59 smithi022 ceph-mon[24777]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-03-21T12:33:01.244 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:00 smithi189 ceph-mon[27080]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-03-21T12:33:01.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:00 smithi022 ceph-mon[24777]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-03-21T12:33:03.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:03 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:33:03.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:03 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:33:03.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:03 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:33:03.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:03 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:33:03.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:03 smithi189 ceph-mon[27080]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-03-21T12:33:03.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:03 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:33:03.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:03 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:33:03.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:03 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:33:03.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:03 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:33:03.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:03 smithi022 ceph-mon[24777]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-03-21T12:33:05.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:04 smithi189 ceph-mon[27080]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-03-21T12:33:05.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:04 smithi022 ceph-mon[24777]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-03-21T12:33:05.548 INFO:teuthology.orchestra.run.smithi189.stdout:Created osd(s) 7 on host 'smithi189' 2024-03-21T12:33:06.131 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:05 smithi189 ceph-mon[27080]: from='osd.7 [v2:172.21.15.189:6824/1090427313,v1:172.21.15.189:6825/1090427313]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-03-21T12:33:06.131 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:05 smithi189 ceph-mon[27080]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-03-21T12:33:06.131 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:05 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:33:06.132 DEBUG:teuthology.orchestra.run.smithi189:osd.7> sudo journalctl -f -n 0 -u ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715@osd.7.service 2024-03-21T12:33:06.135 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-03-21T12:33:06.135 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -- ceph osd stat -f json 2024-03-21T12:33:06.163 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:05 smithi022 ceph-mon[24777]: from='osd.7 [v2:172.21.15.189:6824/1090427313,v1:172.21.15.189:6825/1090427313]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-03-21T12:33:06.163 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:05 smithi022 ceph-mon[24777]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-03-21T12:33:06.163 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:05 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:33:07.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:06 smithi022 ceph-mon[24777]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-03-21T12:33:07.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:06 smithi022 ceph-mon[24777]: osdmap e54: 8 total, 7 up, 8 in 2024-03-21T12:33:07.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:06 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-21T12:33:07.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:06 smithi022 ceph-mon[24777]: from='osd.7 [v2:172.21.15.189:6824/1090427313,v1:172.21.15.189:6825/1090427313]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-03-21T12:33:07.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:06 smithi022 ceph-mon[24777]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-03-21T12:33:07.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:06 smithi022 ceph-mon[24777]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-03-21T12:33:07.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:06 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi022.fektzs/trash_purge_schedule"}]: dispatch 2024-03-21T12:33:07.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:06 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi022.fektzs/mirror_snapshot_schedule"}]: dispatch 2024-03-21T12:33:07.285 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:33:06 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[40286]: 2024-03-21T12:33:06.914+0000 7f0ab50b0700 -1 osd.7 0 waiting for initial osdmap 2024-03-21T12:33:07.286 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:33:06 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[40286]: 2024-03-21T12:33:06.923+0000 7f0aada43700 -1 osd.7 55 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-21T12:33:07.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:06 smithi189 ceph-mon[27080]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-03-21T12:33:07.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:06 smithi189 ceph-mon[27080]: osdmap e54: 8 total, 7 up, 8 in 2024-03-21T12:33:07.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:06 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-21T12:33:07.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:06 smithi189 ceph-mon[27080]: from='osd.7 [v2:172.21.15.189:6824/1090427313,v1:172.21.15.189:6825/1090427313]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-03-21T12:33:07.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:06 smithi189 ceph-mon[27080]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-03-21T12:33:07.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:06 smithi189 ceph-mon[27080]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-03-21T12:33:07.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:06 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi022.fektzs/trash_purge_schedule"}]: dispatch 2024-03-21T12:33:07.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:06 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi022.fektzs/mirror_snapshot_schedule"}]: dispatch 2024-03-21T12:33:08.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:07 smithi189 ceph-mon[27080]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]': finished 2024-03-21T12:33:08.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:07 smithi189 ceph-mon[27080]: osdmap e55: 8 total, 7 up, 8 in 2024-03-21T12:33:08.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:07 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-21T12:33:08.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:07 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-21T12:33:08.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:07 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:33:08.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:07 smithi022 ceph-mon[24777]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]': finished 2024-03-21T12:33:08.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:07 smithi022 ceph-mon[24777]: osdmap e55: 8 total, 7 up, 8 in 2024-03-21T12:33:08.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:07 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-21T12:33:08.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:07 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-21T12:33:08.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:07 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:33:08.197 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-03-21T12:33:08.818 INFO:teuthology.orchestra.run.smithi022.stdout:{"epoch":56,"num_osds":8,"num_up_osds":8,"osd_up_since":1711024387,"num_in_osds":8,"osd_in_since":1711024370,"num_remapped_pgs":1} 2024-03-21T12:33:08.818 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -- ceph osd dump --format=json 2024-03-21T12:33:09.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:08 smithi189 ceph-mon[27080]: purged_snaps scrub starts 2024-03-21T12:33:09.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:08 smithi189 ceph-mon[27080]: purged_snaps scrub ok 2024-03-21T12:33:09.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:08 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-21T12:33:09.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:08 smithi189 ceph-mon[27080]: osd.7 [v2:172.21.15.189:6824/1090427313,v1:172.21.15.189:6825/1090427313] boot 2024-03-21T12:33:09.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:08 smithi189 ceph-mon[27080]: osdmap e56: 8 total, 8 up, 8 in 2024-03-21T12:33:09.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:08 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-21T12:33:09.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:08 smithi189 ceph-mon[27080]: from='client.? 172.21.15.22:0/3346999365' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-03-21T12:33:09.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:08 smithi189 ceph-mon[27080]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-03-21T12:33:09.082 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/mon.smithi022/config 2024-03-21T12:33:09.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:08 smithi022 ceph-mon[24777]: purged_snaps scrub starts 2024-03-21T12:33:09.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:08 smithi022 ceph-mon[24777]: purged_snaps scrub ok 2024-03-21T12:33:09.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:08 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-21T12:33:09.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:08 smithi022 ceph-mon[24777]: osd.7 [v2:172.21.15.189:6824/1090427313,v1:172.21.15.189:6825/1090427313] boot 2024-03-21T12:33:09.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:08 smithi022 ceph-mon[24777]: osdmap e56: 8 total, 8 up, 8 in 2024-03-21T12:33:09.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:08 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-21T12:33:09.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:08 smithi022 ceph-mon[24777]: from='client.? 172.21.15.22:0/3346999365' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-03-21T12:33:09.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:08 smithi022 ceph-mon[24777]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-03-21T12:33:10.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:09 smithi022 ceph-mon[24777]: osdmap e57: 8 total, 8 up, 8 in 2024-03-21T12:33:10.142 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:09 smithi189 ceph-mon[27080]: osdmap e57: 8 total, 8 up, 8 in 2024-03-21T12:33:10.713 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-03-21T12:33:10.714 INFO:teuthology.orchestra.run.smithi022.stdout:{"epoch":58,"fsid":"3fc7f0f2-e77e-11ee-95cd-87774f69a715","created":"2024-03-21T12:27:08.895159+0000","modified":"2024-03-21T12:33:09.919906+0000","last_up_change":"2024-03-21T12:33:07.912052+0000","last_in_change":"2024-03-21T12:32:50.850180+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-21T12:31:09.134334+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"23","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"b91e0a05-db30-4947-ae26-b5e95e7bf8aa","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6802","nonce":1066511009},{"type":"v1","addr":"172.21.15.22:6803","nonce":1066511009}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6804","nonce":1066511009},{"type":"v1","addr":"172.21.15.22:6805","nonce":1066511009}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6808","nonce":1066511009},{"type":"v1","addr":"172.21.15.22:6809","nonce":1066511009}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6806","nonce":1066511009},{"type":"v1","addr":"172.21.15.22:6807","nonce":1066511009}]},"public_addr":"172.21.15.22:6803/1066511009","cluster_addr":"172.21.15.22:6805/1066511009","heartbeat_back_addr":"172.21.15.22:6809/1066511009","heartbeat_front_addr":"172.21.15.22:6807/1066511009","state":["exists","up"]},{"osd":1,"uuid":"4f42c329-33f6-4c0e-9ce5-b8ecc4a3e6f6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":35,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6810","nonce":3661439817},{"type":"v1","addr":"172.21.15.22:6811","nonce":3661439817}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6812","nonce":3661439817},{"type":"v1","addr":"172.21.15.22:6813","nonce":3661439817}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6816","nonce":3661439817},{"type":"v1","addr":"172.21.15.22:6817","nonce":3661439817}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6814","nonce":3661439817},{"type":"v1","addr":"172.21.15.22:6815","nonce":3661439817}]},"public_addr":"172.21.15.22:6811/3661439817","cluster_addr":"172.21.15.22:6813/3661439817","heartbeat_back_addr":"172.21.15.22:6817/3661439817","heartbeat_front_addr":"172.21.15.22:6815/3661439817","state":["exists","up"]},{"osd":2,"uuid":"1efc32fa-d9f6-496b-bc29-6aa4fce1a6ab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6818","nonce":4170559295},{"type":"v1","addr":"172.21.15.22:6819","nonce":4170559295}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6820","nonce":4170559295},{"type":"v1","addr":"172.21.15.22:6821","nonce":4170559295}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6824","nonce":4170559295},{"type":"v1","addr":"172.21.15.22:6825","nonce":4170559295}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6822","nonce":4170559295},{"type":"v1","addr":"172.21.15.22:6823","nonce":4170559295}]},"public_addr":"172.21.15.22:6819/4170559295","cluster_addr":"172.21.15.22:6821/4170559295","heartbeat_back_addr":"172.21.15.22:6825/4170559295","heartbeat_front_addr":"172.21.15.22:6823/4170559295","state":["exists","up"]},{"osd":3,"uuid":"1fb58640-cff6-4041-af18-e3ab287610aa","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":27,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6826","nonce":727407119},{"type":"v1","addr":"172.21.15.22:6827","nonce":727407119}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6828","nonce":727407119},{"type":"v1","addr":"172.21.15.22:6829","nonce":727407119}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6832","nonce":727407119},{"type":"v1","addr":"172.21.15.22:6833","nonce":727407119}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6830","nonce":727407119},{"type":"v1","addr":"172.21.15.22:6831","nonce":727407119}]},"public_addr":"172.21.15.22:6827/727407119","cluster_addr":"172.21.15.22:6829/727407119","heartbeat_back_addr":"172.21.15.22:6833/727407119","heartbeat_front_addr":"172.21.15.22:6831/727407119","state":["exists","up"]},{"osd":4,"uuid":"dc459155-aea6-4910-b1cc-566d824f3e91","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":32,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6800","nonce":1650456785},{"type":"v1","addr":"172.21.15.189:6801","nonce":1650456785}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6802","nonce":1650456785},{"type":"v1","addr":"172.21.15.189:6803","nonce":1650456785}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6806","nonce":1650456785},{"type":"v1","addr":"172.21.15.189:6807","nonce":1650456785}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6804","nonce":1650456785},{"type":"v1","addr":"172.21.15.189:6805","nonce":1650456785}]},"public_addr":"172.21.15.189:6801/1650456785","cluster_addr":"172.21.15.189:6803/1650456785","heartbeat_back_addr":"172.21.15.189:6807/1650456785","heartbeat_front_addr":"172.21.15.189:6805/1650456785","state":["exists","up"]},{"osd":5,"uuid":"3712ad88-b7d4-4b7a-bc34-0ec6bfa92e67","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6808","nonce":2147670707},{"type":"v1","addr":"172.21.15.189:6809","nonce":2147670707}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6810","nonce":2147670707},{"type":"v1","addr":"172.21.15.189:6811","nonce":2147670707}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6814","nonce":2147670707},{"type":"v1","addr":"172.21.15.189:6815","nonce":2147670707}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6812","nonce":2147670707},{"type":"v1","addr":"172.21.15.189:6813","nonce":2147670707}]},"public_addr":"172.21.15.189:6809/2147670707","cluster_addr":"172.21.15.189:6811/2147670707","heartbeat_back_addr":"172.21.15.189:6815/2147670707","heartbeat_front_addr":"172.21.15.189:6813/2147670707","state":["exists","up"]},{"osd":6,"uuid":"fda195d0-e8ef-47eb-9424-6913e213c145","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":51,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6816","nonce":1085921107},{"type":"v1","addr":"172.21.15.189:6817","nonce":1085921107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6818","nonce":1085921107},{"type":"v1","addr":"172.21.15.189:6819","nonce":1085921107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6822","nonce":1085921107},{"type":"v1","addr":"172.21.15.189:6823","nonce":1085921107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6820","nonce":1085921107},{"type":"v1","addr":"172.21.15.189:6821","nonce":1085921107}]},"public_addr":"172.21.15.189:6817/1085921107","cluster_addr":"172.21.15.189:6819/1085921107","heartbeat_back_addr":"172.21.15.189:6823/1085921107","heartbeat_front_addr":"172.21.15.189:6821/1085921107","state":["exists","up"]},{"osd":7,"uuid":"22d112d7-aca4-4a57-bd2a-c5890251bee2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":56,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6824","nonce":1090427313},{"type":"v1","addr":"172.21.15.189:6825","nonce":1090427313}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6826","nonce":1090427313},{"type":"v1","addr":"172.21.15.189:6827","nonce":1090427313}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6830","nonce":1090427313},{"type":"v1","addr":"172.21.15.189:6831","nonce":1090427313}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6828","nonce":1090427313},{"type":"v1","addr":"172.21.15.189:6829","nonce":1090427313}]},"public_addr":"172.21.15.189:6825/1090427313","cluster_addr":"172.21.15.189:6827/1090427313","heartbeat_back_addr":"172.21.15.189:6831/1090427313","heartbeat_front_addr":"172.21.15.189:6829/1090427313","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-21T12:30:08.134522+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-21T12:30:36.910008+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-21T12:31:05.858053+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-21T12:31:31.412607+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-21T12:31:53.517415+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-21T12:32:17.426309+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-21T12:32:42.046164+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-21T12:33:06.175698+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[{"pgid":"1.0","osds":[0,6]}],"primary_temp":[],"blocklist":{"172.21.15.22:0/368809143":"2024-03-22T12:29:06.753399+0000","172.21.15.22:6801/725836069":"2024-03-22T12:29:06.753399+0000","172.21.15.22:6801/1237453012":"2024-03-22T12:28:11.962136+0000","172.21.15.22:0/3267134560":"2024-03-22T12:28:11.962136+0000","172.21.15.22:0/672182801":"2024-03-22T12:28:11.962136+0000","172.21.15.22:0/3009290993":"2024-03-22T12:29:06.753399+0000","172.21.15.22:6800/725836069":"2024-03-22T12:29:06.753399+0000","172.21.15.22:0/1657667047":"2024-03-22T12:28:11.962136+0000","172.21.15.22:0/2763629325":"2024-03-22T12:27:42.530241+0000","172.21.15.22:0/206578323":"2024-03-22T12:27:42.530241+0000","172.21.15.22:6800/1237453012":"2024-03-22T12:28:11.962136+0000","172.21.15.22:0/1678763681":"2024-03-22T12:27:42.530241+0000","172.21.15.22:6800/1323574520":"2024-03-22T12:27:42.530241+0000","172.21.15.22:0/439724682":"2024-03-22T12:29:06.753399+0000","172.21.15.22:6801/1323574520":"2024-03-22T12:27:42.530241+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-21T12:33:10.997 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:10 smithi022 ceph-mon[24777]: osdmap e58: 8 total, 8 up, 8 in 2024-03-21T12:33:10.997 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:10 smithi022 ceph-mon[24777]: Detected new or changed devices on smithi189 2024-03-21T12:33:10.997 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:10 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:33:10.997 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:10 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-21T12:33:10.998 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:10 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-21T12:33:10.998 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:10 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-21T12:33:10.998 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:10 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-21T12:33:10.998 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:10 smithi022 ceph-mon[24777]: Adjusting osd_memory_target on smithi189 to 3984M 2024-03-21T12:33:10.998 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:10 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:33:10.998 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:10 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:33:10.998 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:10 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:33:10.998 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:10 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:33:10.998 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:10 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:33:10.998 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:10 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:33:10.998 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:10 smithi022 ceph-mon[24777]: from='client.? 172.21.15.22:0/2409091170' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-21T12:33:10.998 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:10 smithi022 ceph-mon[24777]: pgmap v168: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-03-21T12:33:11.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:10 smithi189 ceph-mon[27080]: osdmap e58: 8 total, 8 up, 8 in 2024-03-21T12:33:11.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:10 smithi189 ceph-mon[27080]: Detected new or changed devices on smithi189 2024-03-21T12:33:11.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:10 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:33:11.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:10 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-21T12:33:11.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:10 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-21T12:33:11.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:10 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-21T12:33:11.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:10 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-21T12:33:11.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:10 smithi189 ceph-mon[27080]: Adjusting osd_memory_target on smithi189 to 3984M 2024-03-21T12:33:11.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:10 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:33:11.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:10 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:33:11.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:10 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:33:11.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:10 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:33:11.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:10 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:33:11.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:10 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:33:11.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:10 smithi189 ceph-mon[27080]: from='client.? 172.21.15.22:0/2409091170' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-21T12:33:11.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:10 smithi189 ceph-mon[27080]: pgmap v168: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-03-21T12:33:11.346 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-21T12:31:09.134334+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '23', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-03-21T12:33:11.346 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -- ceph osd pool get .mgr pg_num 2024-03-21T12:33:11.567 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/mon.smithi022/config 2024-03-21T12:33:12.009 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:11 smithi022 ceph-mon[24777]: osdmap e59: 8 total, 8 up, 8 in 2024-03-21T12:33:12.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:11 smithi189 ceph-mon[27080]: osdmap e59: 8 total, 8 up, 8 in 2024-03-21T12:33:13.276 INFO:teuthology.orchestra.run.smithi022.stdout:pg_num: 1 2024-03-21T12:33:13.276 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:12 smithi022 ceph-mon[24777]: osdmap e60: 8 total, 8 up, 8 in 2024-03-21T12:33:13.276 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:12 smithi022 ceph-mon[24777]: pgmap v171: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-03-21T12:33:13.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:12 smithi189 ceph-mon[27080]: osdmap e60: 8 total, 8 up, 8 in 2024-03-21T12:33:13.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:12 smithi189 ceph-mon[27080]: pgmap v171: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-03-21T12:33:13.995 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:13 smithi022 ceph-mon[24777]: from='client.? 172.21.15.22:0/2881276373' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-03-21T12:33:13.998 INFO:tasks.cephadm:Setting up client nodes... 2024-03-21T12:33:13.998 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-03-21T12:33:14.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:13 smithi189 ceph-mon[27080]: from='client.? 172.21.15.22:0/2881276373' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-03-21T12:33:15.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:14 smithi189 ceph-mon[27080]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 87 KiB/s, 0 objects/s recovering 2024-03-21T12:33:15.343 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:14 smithi022 ceph-mon[24777]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 87 KiB/s, 0 objects/s recovering 2024-03-21T12:33:16.091 INFO:teuthology.orchestra.run.smithi022.stdout:[client.0] 2024-03-21T12:33:16.091 INFO:teuthology.orchestra.run.smithi022.stdout: key = AQAMKfxlWWRFBRAA+a16q6nsWoF0t5/vItht6Q== 2024-03-21T12:33:16.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:16 smithi022 ceph-mon[24777]: from='client.? 172.21.15.22:0/372852462' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-03-21T12:33:16.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:16 smithi022 ceph-mon[24777]: from='client.? 172.21.15.22:0/372852462' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-03-21T12:33:16.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:16 smithi189 ceph-mon[27080]: from='client.? 172.21.15.22:0/372852462' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-03-21T12:33:16.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:16 smithi189 ceph-mon[27080]: from='client.? 172.21.15.22:0/372852462' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-03-21T12:33:16.784 DEBUG:teuthology.orchestra.run.smithi022:> set -ex 2024-03-21T12:33:16.784 DEBUG:teuthology.orchestra.run.smithi022:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-03-21T12:33:16.784 DEBUG:teuthology.orchestra.run.smithi022:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-03-21T12:33:16.828 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-03-21T12:33:16.828 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-03-21T12:33:16.828 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -- ceph mgr dump --format=json 2024-03-21T12:33:17.084 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/mon.smithi022/config 2024-03-21T12:33:18.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:17 smithi022 ceph-mon[24777]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 75 KiB/s, 0 objects/s recovering 2024-03-21T12:33:18.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:17 smithi189 ceph-mon[27080]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 75 KiB/s, 0 objects/s recovering 2024-03-21T12:33:18.820 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-03-21T12:33:19.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:18 smithi022 ceph-mon[24777]: from='client.? 172.21.15.22:0/2008515117' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-03-21T12:33:19.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:18 smithi189 ceph-mon[27080]: from='client.? 172.21.15.22:0/2008515117' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-03-21T12:33:19.554 INFO:teuthology.orchestra.run.smithi022.stdout:{"epoch":19,"active_gid":14202,"active_name":"smithi022.fektzs","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6800","nonce":3390830032},{"type":"v1","addr":"172.21.15.22:6801","nonce":3390830032}]},"active_addr":"172.21.15.22:6801/3390830032","active_change":"2024-03-21T12:29:06.753854+0000","active_mgr_features":4540138303579357183,"available":true,"standbys":[{"gid":14220,"name":"smithi189.cnxzvu","mgr_features":4540138303579357183,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.22:8443/","prometheus":"http://172.21.15.22:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"last_failure_osd_epoch":5,"active_clients":[{"addrvec":[{"type":"v2","addr":"172.21.15.22:0","nonce":2629817737}]},{"addrvec":[{"type":"v2","addr":"172.21.15.22:0","nonce":3219474775}]},{"addrvec":[{"type":"v2","addr":"172.21.15.22:0","nonce":273928666}]},{"addrvec":[{"type":"v2","addr":"172.21.15.22:0","nonce":2652919562}]}]}} 2024-03-21T12:33:19.561 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-03-21T12:33:19.561 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-03-21T12:33:19.561 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -- ceph osd dump --format=json 2024-03-21T12:33:19.779 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/mon.smithi022/config 2024-03-21T12:33:20.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:19 smithi022 ceph-mon[24777]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-21T12:33:20.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:19 smithi189 ceph-mon[27080]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-21T12:33:21.134 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:20 smithi022 ceph-mon[24777]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-21T12:33:21.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:20 smithi189 ceph-mon[27080]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-21T12:33:21.555 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-03-21T12:33:21.555 INFO:teuthology.orchestra.run.smithi022.stdout:{"epoch":60,"fsid":"3fc7f0f2-e77e-11ee-95cd-87774f69a715","created":"2024-03-21T12:27:08.895159+0000","modified":"2024-03-21T12:33:11.929328+0000","last_up_change":"2024-03-21T12:33:07.912052+0000","last_in_change":"2024-03-21T12:32:50.850180+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-21T12:31:09.134334+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"23","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"b91e0a05-db30-4947-ae26-b5e95e7bf8aa","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6802","nonce":1066511009},{"type":"v1","addr":"172.21.15.22:6803","nonce":1066511009}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6804","nonce":1066511009},{"type":"v1","addr":"172.21.15.22:6805","nonce":1066511009}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6808","nonce":1066511009},{"type":"v1","addr":"172.21.15.22:6809","nonce":1066511009}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6806","nonce":1066511009},{"type":"v1","addr":"172.21.15.22:6807","nonce":1066511009}]},"public_addr":"172.21.15.22:6803/1066511009","cluster_addr":"172.21.15.22:6805/1066511009","heartbeat_back_addr":"172.21.15.22:6809/1066511009","heartbeat_front_addr":"172.21.15.22:6807/1066511009","state":["exists","up"]},{"osd":1,"uuid":"4f42c329-33f6-4c0e-9ce5-b8ecc4a3e6f6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":35,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6810","nonce":3661439817},{"type":"v1","addr":"172.21.15.22:6811","nonce":3661439817}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6812","nonce":3661439817},{"type":"v1","addr":"172.21.15.22:6813","nonce":3661439817}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6816","nonce":3661439817},{"type":"v1","addr":"172.21.15.22:6817","nonce":3661439817}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6814","nonce":3661439817},{"type":"v1","addr":"172.21.15.22:6815","nonce":3661439817}]},"public_addr":"172.21.15.22:6811/3661439817","cluster_addr":"172.21.15.22:6813/3661439817","heartbeat_back_addr":"172.21.15.22:6817/3661439817","heartbeat_front_addr":"172.21.15.22:6815/3661439817","state":["exists","up"]},{"osd":2,"uuid":"1efc32fa-d9f6-496b-bc29-6aa4fce1a6ab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6818","nonce":4170559295},{"type":"v1","addr":"172.21.15.22:6819","nonce":4170559295}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6820","nonce":4170559295},{"type":"v1","addr":"172.21.15.22:6821","nonce":4170559295}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6824","nonce":4170559295},{"type":"v1","addr":"172.21.15.22:6825","nonce":4170559295}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6822","nonce":4170559295},{"type":"v1","addr":"172.21.15.22:6823","nonce":4170559295}]},"public_addr":"172.21.15.22:6819/4170559295","cluster_addr":"172.21.15.22:6821/4170559295","heartbeat_back_addr":"172.21.15.22:6825/4170559295","heartbeat_front_addr":"172.21.15.22:6823/4170559295","state":["exists","up"]},{"osd":3,"uuid":"1fb58640-cff6-4041-af18-e3ab287610aa","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":27,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6826","nonce":727407119},{"type":"v1","addr":"172.21.15.22:6827","nonce":727407119}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6828","nonce":727407119},{"type":"v1","addr":"172.21.15.22:6829","nonce":727407119}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6832","nonce":727407119},{"type":"v1","addr":"172.21.15.22:6833","nonce":727407119}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6830","nonce":727407119},{"type":"v1","addr":"172.21.15.22:6831","nonce":727407119}]},"public_addr":"172.21.15.22:6827/727407119","cluster_addr":"172.21.15.22:6829/727407119","heartbeat_back_addr":"172.21.15.22:6833/727407119","heartbeat_front_addr":"172.21.15.22:6831/727407119","state":["exists","up"]},{"osd":4,"uuid":"dc459155-aea6-4910-b1cc-566d824f3e91","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":32,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6800","nonce":1650456785},{"type":"v1","addr":"172.21.15.189:6801","nonce":1650456785}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6802","nonce":1650456785},{"type":"v1","addr":"172.21.15.189:6803","nonce":1650456785}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6806","nonce":1650456785},{"type":"v1","addr":"172.21.15.189:6807","nonce":1650456785}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6804","nonce":1650456785},{"type":"v1","addr":"172.21.15.189:6805","nonce":1650456785}]},"public_addr":"172.21.15.189:6801/1650456785","cluster_addr":"172.21.15.189:6803/1650456785","heartbeat_back_addr":"172.21.15.189:6807/1650456785","heartbeat_front_addr":"172.21.15.189:6805/1650456785","state":["exists","up"]},{"osd":5,"uuid":"3712ad88-b7d4-4b7a-bc34-0ec6bfa92e67","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6808","nonce":2147670707},{"type":"v1","addr":"172.21.15.189:6809","nonce":2147670707}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6810","nonce":2147670707},{"type":"v1","addr":"172.21.15.189:6811","nonce":2147670707}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6814","nonce":2147670707},{"type":"v1","addr":"172.21.15.189:6815","nonce":2147670707}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6812","nonce":2147670707},{"type":"v1","addr":"172.21.15.189:6813","nonce":2147670707}]},"public_addr":"172.21.15.189:6809/2147670707","cluster_addr":"172.21.15.189:6811/2147670707","heartbeat_back_addr":"172.21.15.189:6815/2147670707","heartbeat_front_addr":"172.21.15.189:6813/2147670707","state":["exists","up"]},{"osd":6,"uuid":"fda195d0-e8ef-47eb-9424-6913e213c145","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":51,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6816","nonce":1085921107},{"type":"v1","addr":"172.21.15.189:6817","nonce":1085921107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6818","nonce":1085921107},{"type":"v1","addr":"172.21.15.189:6819","nonce":1085921107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6822","nonce":1085921107},{"type":"v1","addr":"172.21.15.189:6823","nonce":1085921107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6820","nonce":1085921107},{"type":"v1","addr":"172.21.15.189:6821","nonce":1085921107}]},"public_addr":"172.21.15.189:6817/1085921107","cluster_addr":"172.21.15.189:6819/1085921107","heartbeat_back_addr":"172.21.15.189:6823/1085921107","heartbeat_front_addr":"172.21.15.189:6821/1085921107","state":["exists","up"]},{"osd":7,"uuid":"22d112d7-aca4-4a57-bd2a-c5890251bee2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":56,"up_thru":59,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6824","nonce":1090427313},{"type":"v1","addr":"172.21.15.189:6825","nonce":1090427313}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6826","nonce":1090427313},{"type":"v1","addr":"172.21.15.189:6827","nonce":1090427313}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6830","nonce":1090427313},{"type":"v1","addr":"172.21.15.189:6831","nonce":1090427313}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6828","nonce":1090427313},{"type":"v1","addr":"172.21.15.189:6829","nonce":1090427313}]},"public_addr":"172.21.15.189:6825/1090427313","cluster_addr":"172.21.15.189:6827/1090427313","heartbeat_back_addr":"172.21.15.189:6831/1090427313","heartbeat_front_addr":"172.21.15.189:6829/1090427313","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-21T12:30:08.134522+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-21T12:30:36.910008+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-21T12:31:05.858053+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-21T12:31:31.412607+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-21T12:31:53.517415+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-21T12:32:17.426309+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-21T12:32:42.046164+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-21T12:33:06.175698+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.22:0/368809143":"2024-03-22T12:29:06.753399+0000","172.21.15.22:6801/725836069":"2024-03-22T12:29:06.753399+0000","172.21.15.22:6801/1237453012":"2024-03-22T12:28:11.962136+0000","172.21.15.22:0/3267134560":"2024-03-22T12:28:11.962136+0000","172.21.15.22:0/672182801":"2024-03-22T12:28:11.962136+0000","172.21.15.22:0/3009290993":"2024-03-22T12:29:06.753399+0000","172.21.15.22:6800/725836069":"2024-03-22T12:29:06.753399+0000","172.21.15.22:0/1657667047":"2024-03-22T12:28:11.962136+0000","172.21.15.22:0/2763629325":"2024-03-22T12:27:42.530241+0000","172.21.15.22:0/206578323":"2024-03-22T12:27:42.530241+0000","172.21.15.22:6800/1237453012":"2024-03-22T12:28:11.962136+0000","172.21.15.22:0/1678763681":"2024-03-22T12:27:42.530241+0000","172.21.15.22:6800/1323574520":"2024-03-22T12:27:42.530241+0000","172.21.15.22:0/439724682":"2024-03-22T12:29:06.753399+0000","172.21.15.22:6801/1323574520":"2024-03-22T12:27:42.530241+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-21T12:33:22.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:21 smithi189 ceph-mon[27080]: from='client.? 172.21.15.22:0/1905861953' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-21T12:33:22.278 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:21 smithi022 ceph-mon[24777]: from='client.? 172.21.15.22:0/1905861953' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-21T12:33:22.303 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-03-21T12:33:22.303 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -- ceph osd dump --format=json 2024-03-21T12:33:22.524 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/mon.smithi022/config 2024-03-21T12:33:23.167 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:22 smithi022 ceph-mon[24777]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-21T12:33:23.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:22 smithi189 ceph-mon[27080]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-21T12:33:24.189 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-03-21T12:33:24.189 INFO:teuthology.orchestra.run.smithi022.stdout:{"epoch":60,"fsid":"3fc7f0f2-e77e-11ee-95cd-87774f69a715","created":"2024-03-21T12:27:08.895159+0000","modified":"2024-03-21T12:33:11.929328+0000","last_up_change":"2024-03-21T12:33:07.912052+0000","last_in_change":"2024-03-21T12:32:50.850180+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-21T12:31:09.134334+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"23","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"b91e0a05-db30-4947-ae26-b5e95e7bf8aa","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6802","nonce":1066511009},{"type":"v1","addr":"172.21.15.22:6803","nonce":1066511009}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6804","nonce":1066511009},{"type":"v1","addr":"172.21.15.22:6805","nonce":1066511009}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6808","nonce":1066511009},{"type":"v1","addr":"172.21.15.22:6809","nonce":1066511009}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6806","nonce":1066511009},{"type":"v1","addr":"172.21.15.22:6807","nonce":1066511009}]},"public_addr":"172.21.15.22:6803/1066511009","cluster_addr":"172.21.15.22:6805/1066511009","heartbeat_back_addr":"172.21.15.22:6809/1066511009","heartbeat_front_addr":"172.21.15.22:6807/1066511009","state":["exists","up"]},{"osd":1,"uuid":"4f42c329-33f6-4c0e-9ce5-b8ecc4a3e6f6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":35,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6810","nonce":3661439817},{"type":"v1","addr":"172.21.15.22:6811","nonce":3661439817}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6812","nonce":3661439817},{"type":"v1","addr":"172.21.15.22:6813","nonce":3661439817}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6816","nonce":3661439817},{"type":"v1","addr":"172.21.15.22:6817","nonce":3661439817}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6814","nonce":3661439817},{"type":"v1","addr":"172.21.15.22:6815","nonce":3661439817}]},"public_addr":"172.21.15.22:6811/3661439817","cluster_addr":"172.21.15.22:6813/3661439817","heartbeat_back_addr":"172.21.15.22:6817/3661439817","heartbeat_front_addr":"172.21.15.22:6815/3661439817","state":["exists","up"]},{"osd":2,"uuid":"1efc32fa-d9f6-496b-bc29-6aa4fce1a6ab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6818","nonce":4170559295},{"type":"v1","addr":"172.21.15.22:6819","nonce":4170559295}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6820","nonce":4170559295},{"type":"v1","addr":"172.21.15.22:6821","nonce":4170559295}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6824","nonce":4170559295},{"type":"v1","addr":"172.21.15.22:6825","nonce":4170559295}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6822","nonce":4170559295},{"type":"v1","addr":"172.21.15.22:6823","nonce":4170559295}]},"public_addr":"172.21.15.22:6819/4170559295","cluster_addr":"172.21.15.22:6821/4170559295","heartbeat_back_addr":"172.21.15.22:6825/4170559295","heartbeat_front_addr":"172.21.15.22:6823/4170559295","state":["exists","up"]},{"osd":3,"uuid":"1fb58640-cff6-4041-af18-e3ab287610aa","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":27,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6826","nonce":727407119},{"type":"v1","addr":"172.21.15.22:6827","nonce":727407119}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6828","nonce":727407119},{"type":"v1","addr":"172.21.15.22:6829","nonce":727407119}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6832","nonce":727407119},{"type":"v1","addr":"172.21.15.22:6833","nonce":727407119}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6830","nonce":727407119},{"type":"v1","addr":"172.21.15.22:6831","nonce":727407119}]},"public_addr":"172.21.15.22:6827/727407119","cluster_addr":"172.21.15.22:6829/727407119","heartbeat_back_addr":"172.21.15.22:6833/727407119","heartbeat_front_addr":"172.21.15.22:6831/727407119","state":["exists","up"]},{"osd":4,"uuid":"dc459155-aea6-4910-b1cc-566d824f3e91","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":32,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6800","nonce":1650456785},{"type":"v1","addr":"172.21.15.189:6801","nonce":1650456785}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6802","nonce":1650456785},{"type":"v1","addr":"172.21.15.189:6803","nonce":1650456785}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6806","nonce":1650456785},{"type":"v1","addr":"172.21.15.189:6807","nonce":1650456785}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6804","nonce":1650456785},{"type":"v1","addr":"172.21.15.189:6805","nonce":1650456785}]},"public_addr":"172.21.15.189:6801/1650456785","cluster_addr":"172.21.15.189:6803/1650456785","heartbeat_back_addr":"172.21.15.189:6807/1650456785","heartbeat_front_addr":"172.21.15.189:6805/1650456785","state":["exists","up"]},{"osd":5,"uuid":"3712ad88-b7d4-4b7a-bc34-0ec6bfa92e67","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6808","nonce":2147670707},{"type":"v1","addr":"172.21.15.189:6809","nonce":2147670707}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6810","nonce":2147670707},{"type":"v1","addr":"172.21.15.189:6811","nonce":2147670707}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6814","nonce":2147670707},{"type":"v1","addr":"172.21.15.189:6815","nonce":2147670707}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6812","nonce":2147670707},{"type":"v1","addr":"172.21.15.189:6813","nonce":2147670707}]},"public_addr":"172.21.15.189:6809/2147670707","cluster_addr":"172.21.15.189:6811/2147670707","heartbeat_back_addr":"172.21.15.189:6815/2147670707","heartbeat_front_addr":"172.21.15.189:6813/2147670707","state":["exists","up"]},{"osd":6,"uuid":"fda195d0-e8ef-47eb-9424-6913e213c145","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":51,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6816","nonce":1085921107},{"type":"v1","addr":"172.21.15.189:6817","nonce":1085921107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6818","nonce":1085921107},{"type":"v1","addr":"172.21.15.189:6819","nonce":1085921107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6822","nonce":1085921107},{"type":"v1","addr":"172.21.15.189:6823","nonce":1085921107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6820","nonce":1085921107},{"type":"v1","addr":"172.21.15.189:6821","nonce":1085921107}]},"public_addr":"172.21.15.189:6817/1085921107","cluster_addr":"172.21.15.189:6819/1085921107","heartbeat_back_addr":"172.21.15.189:6823/1085921107","heartbeat_front_addr":"172.21.15.189:6821/1085921107","state":["exists","up"]},{"osd":7,"uuid":"22d112d7-aca4-4a57-bd2a-c5890251bee2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":56,"up_thru":59,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6824","nonce":1090427313},{"type":"v1","addr":"172.21.15.189:6825","nonce":1090427313}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6826","nonce":1090427313},{"type":"v1","addr":"172.21.15.189:6827","nonce":1090427313}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6830","nonce":1090427313},{"type":"v1","addr":"172.21.15.189:6831","nonce":1090427313}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6828","nonce":1090427313},{"type":"v1","addr":"172.21.15.189:6829","nonce":1090427313}]},"public_addr":"172.21.15.189:6825/1090427313","cluster_addr":"172.21.15.189:6827/1090427313","heartbeat_back_addr":"172.21.15.189:6831/1090427313","heartbeat_front_addr":"172.21.15.189:6829/1090427313","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-21T12:30:08.134522+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-21T12:30:36.910008+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-21T12:31:05.858053+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-21T12:31:31.412607+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-21T12:31:53.517415+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-21T12:32:17.426309+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-21T12:32:42.046164+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-21T12:33:06.175698+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.22:0/368809143":"2024-03-22T12:29:06.753399+0000","172.21.15.22:6801/725836069":"2024-03-22T12:29:06.753399+0000","172.21.15.22:6801/1237453012":"2024-03-22T12:28:11.962136+0000","172.21.15.22:0/3267134560":"2024-03-22T12:28:11.962136+0000","172.21.15.22:0/672182801":"2024-03-22T12:28:11.962136+0000","172.21.15.22:0/3009290993":"2024-03-22T12:29:06.753399+0000","172.21.15.22:6800/725836069":"2024-03-22T12:29:06.753399+0000","172.21.15.22:0/1657667047":"2024-03-22T12:28:11.962136+0000","172.21.15.22:0/2763629325":"2024-03-22T12:27:42.530241+0000","172.21.15.22:0/206578323":"2024-03-22T12:27:42.530241+0000","172.21.15.22:6800/1237453012":"2024-03-22T12:28:11.962136+0000","172.21.15.22:0/1678763681":"2024-03-22T12:27:42.530241+0000","172.21.15.22:6800/1323574520":"2024-03-22T12:27:42.530241+0000","172.21.15.22:0/439724682":"2024-03-22T12:29:06.753399+0000","172.21.15.22:6801/1323574520":"2024-03-22T12:27:42.530241+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-21T12:33:24.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:24 smithi022 ceph-mon[24777]: from='client.? 172.21.15.22:0/46201978' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-21T12:33:24.532 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:24 smithi189 ceph-mon[27080]: from='client.? 172.21.15.22:0/46201978' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-21T12:33:24.815 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -- ceph tell osd.0 flush_pg_stats 2024-03-21T12:33:24.816 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -- ceph tell osd.1 flush_pg_stats 2024-03-21T12:33:24.816 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -- ceph tell osd.2 flush_pg_stats 2024-03-21T12:33:24.817 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -- ceph tell osd.3 flush_pg_stats 2024-03-21T12:33:24.817 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -- ceph tell osd.4 flush_pg_stats 2024-03-21T12:33:24.818 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -- ceph tell osd.5 flush_pg_stats 2024-03-21T12:33:24.818 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -- ceph tell osd.6 flush_pg_stats 2024-03-21T12:33:24.818 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -- ceph tell osd.7 flush_pg_stats 2024-03-21T12:33:25.235 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/mon.smithi022/config 2024-03-21T12:33:25.235 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/mon.smithi022/config 2024-03-21T12:33:25.260 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/mon.smithi022/config 2024-03-21T12:33:25.260 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/mon.smithi022/config 2024-03-21T12:33:25.262 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/mon.smithi022/config 2024-03-21T12:33:25.282 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/mon.smithi022/config 2024-03-21T12:33:25.374 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:25 smithi022 ceph-mon[24777]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-21T12:33:25.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:25 smithi189 ceph-mon[27080]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-21T12:33:26.153 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/mon.smithi022/config 2024-03-21T12:33:26.153 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/mon.smithi022/config 2024-03-21T12:33:28.039 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:27 smithi022 ceph-mon[24777]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-21T12:33:28.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:27 smithi189 ceph-mon[27080]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-21T12:33:29.128 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:28 smithi022 ceph-mon[24777]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-21T12:33:29.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:28 smithi189 ceph-mon[27080]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-21T12:33:31.195 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:30 smithi022 ceph-mon[24777]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-21T12:33:31.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:30 smithi189 ceph-mon[27080]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-21T12:33:33.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:32 smithi189 ceph-mon[27080]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-21T12:33:33.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:32 smithi022 ceph-mon[24777]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-21T12:33:33.937 INFO:teuthology.orchestra.run.smithi022.stdout:81604378655 2024-03-21T12:33:33.938 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -- ceph osd last-stat-seq osd.2 2024-03-21T12:33:35.081 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:34 smithi022 ceph-mon[24777]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-21T12:33:35.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:34 smithi189 ceph-mon[27080]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-21T12:33:35.450 INFO:teuthology.orchestra.run.smithi022.stdout:38654705708 2024-03-21T12:33:35.451 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -- ceph osd last-stat-seq osd.0 2024-03-21T12:33:36.181 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/mon.smithi022/config 2024-03-21T12:33:37.717 INFO:teuthology.orchestra.run.smithi022.stdout:171798691859 2024-03-21T12:33:37.718 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -- ceph osd last-stat-seq osd.5 2024-03-21T12:33:37.997 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:37 smithi022 ceph-mon[24777]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-21T12:33:38.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:37 smithi189 ceph-mon[27080]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-21T12:33:39.003 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:38 smithi022 ceph-mon[24777]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-21T12:33:39.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:38 smithi189 ceph-mon[27080]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-21T12:33:40.854 INFO:teuthology.orchestra.run.smithi022.stdout:115964117019 2024-03-21T12:33:40.854 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -- ceph osd last-stat-seq osd.3 2024-03-21T12:33:41.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:40 smithi022 ceph-mon[24777]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-21T12:33:41.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:40 smithi189 ceph-mon[27080]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-21T12:33:41.565 INFO:teuthology.orchestra.run.smithi022.stdout:137438953496 2024-03-21T12:33:41.566 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -- ceph osd last-stat-seq osd.4 2024-03-21T12:33:42.181 INFO:teuthology.orchestra.run.smithi022.stdout:240518168586 2024-03-21T12:33:42.181 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -- ceph osd last-stat-seq osd.7 2024-03-21T12:33:42.876 INFO:teuthology.orchestra.run.smithi022.stdout:206158430221 2024-03-21T12:33:42.876 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -- ceph osd last-stat-seq osd.6 2024-03-21T12:33:43.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:42 smithi022 ceph-mon[24777]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-21T12:33:43.278 INFO:teuthology.orchestra.run.smithi022.stdout:60129542182 2024-03-21T12:33:43.278 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -- ceph osd last-stat-seq osd.1 2024-03-21T12:33:43.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:42 smithi189 ceph-mon[27080]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-21T12:33:43.553 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/mon.smithi022/config 2024-03-21T12:33:43.554 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/mon.smithi022/config 2024-03-21T12:33:43.556 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/mon.smithi022/config 2024-03-21T12:33:43.556 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/mon.smithi022/config 2024-03-21T12:33:43.577 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/mon.smithi022/config 2024-03-21T12:33:43.682 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/mon.smithi022/config 2024-03-21T12:33:43.978 INFO:teuthology.orchestra.run.smithi022.stdout:81604378657 2024-03-21T12:33:44.171 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:44 smithi022 ceph-mon[24777]: from='client.? 172.21.15.22:0/3260377203' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-03-21T12:33:44.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:44 smithi189 ceph-mon[27080]: from='client.? 172.21.15.22:0/3260377203' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-03-21T12:33:45.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:45 smithi189 ceph-mon[27080]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-21T12:33:45.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:45 smithi022 ceph-mon[24777]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-21T12:33:45.637 INFO:tasks.cephadm.ceph_manager.ceph:need seq 81604378655 got 81604378657 for osd.2 2024-03-21T12:33:45.637 DEBUG:teuthology.parallel:result is None 2024-03-21T12:33:47.897 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:47 smithi022 ceph-mon[24777]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-21T12:33:48.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:47 smithi189 ceph-mon[27080]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-21T12:33:48.502 INFO:teuthology.orchestra.run.smithi022.stdout:115964117021 2024-03-21T12:33:49.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:48 smithi022 ceph-mon[24777]: from='client.? 172.21.15.22:0/1353390901' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-03-21T12:33:49.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:48 smithi189 ceph-mon[27080]: from='client.? 172.21.15.22:0/1353390901' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-03-21T12:33:49.927 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:49 smithi022 ceph-mon[24777]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-21T12:33:49.954 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/mon.smithi022/config 2024-03-21T12:33:49.957 INFO:tasks.cephadm.ceph_manager.ceph:need seq 115964117019 got 115964117021 for osd.3 2024-03-21T12:33:49.957 DEBUG:teuthology.parallel:result is None 2024-03-21T12:33:50.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:49 smithi189 ceph-mon[27080]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-21T12:33:50.694 INFO:teuthology.orchestra.run.smithi022.stdout:240518168587 2024-03-21T12:33:50.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:50 smithi022 ceph-mon[24777]: from='client.? 172.21.15.22:0/1416668452' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-03-21T12:33:51.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:50 smithi189 ceph-mon[27080]: from='client.? 172.21.15.22:0/1416668452' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-03-21T12:33:51.751 INFO:tasks.cephadm.ceph_manager.ceph:need seq 240518168586 got 240518168587 for osd.7 2024-03-21T12:33:51.751 DEBUG:teuthology.parallel:result is None 2024-03-21T12:33:51.977 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:51 smithi022 ceph-mon[24777]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-21T12:33:52.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:51 smithi189 ceph-mon[27080]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-21T12:33:52.579 INFO:teuthology.orchestra.run.smithi022.stdout:171798691862 2024-03-21T12:33:52.768 INFO:teuthology.orchestra.run.smithi022.stdout:38654705712 2024-03-21T12:33:53.109 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:52 smithi022 ceph-mon[24777]: from='client.? 172.21.15.22:0/3799556439' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-03-21T12:33:53.110 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:52 smithi022 ceph-mon[24777]: from='client.? 172.21.15.22:0/3527136756' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-03-21T12:33:53.275 INFO:teuthology.orchestra.run.smithi022.stdout:137438953499 2024-03-21T12:33:53.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:52 smithi189 ceph-mon[27080]: from='client.? 172.21.15.22:0/3799556439' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-03-21T12:33:53.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:52 smithi189 ceph-mon[27080]: from='client.? 172.21.15.22:0/3527136756' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-03-21T12:33:53.829 INFO:tasks.cephadm.ceph_manager.ceph:need seq 171798691859 got 171798691862 for osd.5 2024-03-21T12:33:53.829 DEBUG:teuthology.parallel:result is None 2024-03-21T12:33:54.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:53 smithi022 ceph-mon[24777]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-21T12:33:54.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:53 smithi022 ceph-mon[24777]: from='client.? 172.21.15.22:0/1169940679' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-03-21T12:33:54.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:53 smithi189 ceph-mon[27080]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-21T12:33:54.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:53 smithi189 ceph-mon[27080]: from='client.? 172.21.15.22:0/1169940679' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-03-21T12:33:55.058 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:54 smithi022 ceph-mon[24777]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-21T12:33:55.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:54 smithi189 ceph-mon[27080]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-21T12:33:55.462 INFO:tasks.cephadm.ceph_manager.ceph:need seq 38654705708 got 38654705712 for osd.0 2024-03-21T12:33:55.462 DEBUG:teuthology.parallel:result is None 2024-03-21T12:33:56.475 INFO:tasks.cephadm.ceph_manager.ceph:need seq 137438953496 got 137438953499 for osd.4 2024-03-21T12:33:56.476 DEBUG:teuthology.parallel:result is None 2024-03-21T12:33:56.899 INFO:teuthology.orchestra.run.smithi022.stdout:206158430224 2024-03-21T12:33:57.982 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:57 smithi022 ceph-mon[24777]: pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-21T12:33:57.982 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:57 smithi022 ceph-mon[24777]: from='client.? 172.21.15.22:0/3675378277' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-03-21T12:33:58.008 INFO:tasks.cephadm.ceph_manager.ceph:need seq 206158430221 got 206158430224 for osd.6 2024-03-21T12:33:58.008 DEBUG:teuthology.parallel:result is None 2024-03-21T12:33:58.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:57 smithi189 ceph-mon[27080]: pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-21T12:33:58.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:57 smithi189 ceph-mon[27080]: from='client.? 172.21.15.22:0/3675378277' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-03-21T12:33:58.432 INFO:teuthology.orchestra.run.smithi022.stdout:60129542186 2024-03-21T12:33:59.020 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:33:58 smithi022 ceph-mon[24777]: from='client.? 172.21.15.22:0/2862031187' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-03-21T12:33:59.046 INFO:tasks.cephadm.ceph_manager.ceph:need seq 60129542182 got 60129542186 for osd.1 2024-03-21T12:33:59.046 DEBUG:teuthology.parallel:result is None 2024-03-21T12:33:59.046 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-03-21T12:33:59.047 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -- ceph pg dump --format=json 2024-03-21T12:33:59.270 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/mon.smithi022/config 2024-03-21T12:33:59.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:33:58 smithi189 ceph-mon[27080]: from='client.? 172.21.15.22:0/2862031187' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-03-21T12:34:00.932 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-03-21T12:34:00.933 INFO:teuthology.orchestra.run.smithi022.stderr:dumped all 2024-03-21T12:34:01.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:00 smithi022 ceph-mon[24777]: pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-21T12:34:01.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:00 smithi189 ceph-mon[27080]: pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-21T12:34:01.551 INFO:teuthology.orchestra.run.smithi022.stdout:{"pg_ready":true,"pg_map":{"version":195,"stamp":"2024-03-21T12:34:00.856075+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48880,"kb_used_data":6384,"kb_used_omap":0,"kb_used_meta":42432,"kb_avail":749912336,"statfs":{"total":767960285184,"available":767910232064,"internally_reserved":0,"allocated":6537216,"data_stored":3571908,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43450368},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002001"},"pg_stats":[{"pgid":"1.0","version":"23'117","reported_seq":226,"reported_epoch":60,"state":"active+clean","last_fresh":"2024-03-21T12:33:11.938599+0000","last_change":"2024-03-21T12:33:11.938599+0000","last_active":"2024-03-21T12:33:11.938599+0000","last_peered":"2024-03-21T12:33:11.938599+0000","last_clean":"2024-03-21T12:33:11.938599+0000","last_became_active":"2024-03-21T12:33:11.938228+0000","last_became_peered":"2024-03-21T12:33:11.938228+0000","last_unstale":"2024-03-21T12:33:11.938599+0000","last_undegraded":"2024-03-21T12:33:11.938599+0000","last_fullsized":"2024-03-21T12:33:11.938599+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":21,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-21T12:31:09.463044+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-21T12:31:09.463044+0000","last_clean_scrub_stamp":"2024-03-21T12:31:09.463044+0000","objects_scrubbed":0,"log_size":117,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-22T13:40:33.828750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3727360,"data_stored":3702496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":56,"seq":240518168590,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6060,"kb_used_data":1124,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739092,"statfs":{"total":95995035648,"available":95988830208,"internally_reserved":0,"allocated":1150976,"data_stored":777142,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.52800000000000002}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.59299999999999997}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51800000000000002}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.46800000000000003}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.33800000000000002}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.48099999999999998}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54000000000000004}]}]},{"osd":6,"up_from":48,"seq":206158430226,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6180,"kb_used_data":1116,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93738972,"statfs":{"total":95995035648,"available":95988707328,"internally_reserved":0,"allocated":1142784,"data_stored":776554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Mar 21 12:33:46 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.68899999999999995,"5min":0.68899999999999995,"15min":0.68899999999999995},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.51600000000000001}]},{"osd":1,"last update":"Thu Mar 21 12:33:46 2024","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.81399999999999995,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.41199999999999998}]},{"osd":2,"last update":"Thu Mar 21 12:33:46 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.35999999999999999},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.439}]},{"osd":3,"last update":"Thu Mar 21 12:33:46 2024","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.78100000000000003,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"last":0.47199999999999998}]},{"osd":4,"last update":"Thu Mar 21 12:33:46 2024","interfaces":[{"interface":"back","average":{"1min":0.432,"5min":0.432,"15min":0.432},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.375}]},{"osd":5,"last update":"Thu Mar 21 12:33:46 2024","interfaces":[{"interface":"back","average":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":0.39900000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45200000000000001}]}]},{"osd":1,"up_from":14,"seq":60129542187,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6372,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738780,"statfs":{"total":95995035648,"available":95988510720,"internally_reserved":0,"allocated":618496,"data_stored":248214,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Mar 21 12:33:40 2024","interfaces":[{"interface":"back","average":{"1min":0.45200000000000001,"5min":0.36499999999999999,"15min":0.34200000000000003},"min":{"1min":0.25700000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":1.661,"5min":1.661,"15min":1.661},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.40100000000000002,"5min":0.34599999999999997,"15min":0.32500000000000001},"min":{"1min":0.27400000000000002,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.38700000000000001}]},{"osd":2,"last update":"Thu Mar 21 12:33:14 2024","interfaces":[{"interface":"back","average":{"1min":0.39400000000000002,"5min":0.38400000000000001,"15min":0.38200000000000001},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.53200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.39400000000000002,"5min":0.378,"15min":0.375},"min":{"1min":0.28699999999999998,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.54200000000000004,"5min":0.64400000000000002,"15min":0.64400000000000002},"last":0.41399999999999998}]},{"osd":3,"last update":"Thu Mar 21 12:33:40 2024","interfaces":[{"interface":"back","average":{"1min":0.495,"5min":0.45100000000000001,"15min":0.443},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":1.7370000000000001,"5min":1.7370000000000001,"15min":1.7370000000000001},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.5,"5min":0.45600000000000002,"15min":0.44900000000000001},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":1.595,"5min":1.595,"15min":1.595},"last":0.47699999999999998}]},{"osd":4,"last update":"Thu Mar 21 12:33:59 2024","interfaces":[{"interface":"back","average":{"1min":0.83399999999999996,"5min":0.56200000000000006,"15min":0.51700000000000002},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":5.2409999999999997,"5min":5.2409999999999997,"15min":5.2409999999999997},"last":0.49299999999999999},{"interface":"front","average":{"1min":0.65800000000000003,"5min":0.55400000000000005,"15min":0.53700000000000003},"min":{"1min":0.441,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":1.7629999999999999,"5min":1.7629999999999999,"15min":1.7629999999999999},"last":0.54800000000000004}]},{"osd":5,"last update":"Thu Mar 21 12:33:20 2024","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.504},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.53300000000000003}]},{"osd":6,"last update":"Thu Mar 21 12:33:45 2024","interfaces":[{"interface":"back","average":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":1.877,"5min":1.877,"15min":1.877},"last":0.65000000000000002},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.56799999999999995}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.59999999999999998}]}]},{"osd":0,"up_from":9,"seq":38654705714,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6892,"kb_used_data":1124,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738260,"statfs":{"total":95995035648,"available":95987978240,"internally_reserved":0,"allocated":1150976,"data_stored":777142,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Mar 21 12:33:43 2024","interfaces":[{"interface":"back","average":{"1min":0.42299999999999999,"5min":0.34499999999999997,"15min":0.317},"min":{"1min":0.252,"5min":0.214,"15min":0.214},"max":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.44900000000000001,"5min":0.36599999999999999,"15min":0.33300000000000002},"min":{"1min":0.25900000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.54600000000000004}]},{"osd":2,"last update":"Thu Mar 21 12:33:13 2024","interfaces":[{"interface":"back","average":{"1min":0.42199999999999999,"5min":0.40200000000000002,"15min":0.39800000000000002},"min":{"1min":0.27300000000000002,"5min":0.25,"15min":0.25},"max":{"1min":0.61899999999999999,"5min":0.89400000000000002,"15min":0.89400000000000002},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.44700000000000001,"5min":0.39600000000000002,"15min":0.38800000000000001},"min":{"1min":0.34200000000000003,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"last":0.48299999999999998}]},{"osd":3,"last update":"Thu Mar 21 12:33:33 2024","interfaces":[{"interface":"back","average":{"1min":0.42899999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"min":{"1min":0.26600000000000001,"5min":0.253,"15min":0.253},"max":{"1min":0.55600000000000005,"5min":0.63600000000000001,"15min":0.63600000000000001},"last":0.432},{"interface":"front","average":{"1min":0.501,"5min":0.46899999999999997,"15min":0.46300000000000002},"min":{"1min":0.33500000000000002,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.52900000000000003}]},{"osd":4,"last update":"Thu Mar 21 12:34:00 2024","interfaces":[{"interface":"back","average":{"1min":0.64300000000000002,"5min":0.60999999999999999,"15min":0.60399999999999998},"min":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":0.81999999999999995,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.72599999999999998},{"interface":"front","average":{"1min":0.66500000000000004,"5min":0.627,"15min":0.621},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.872,"5min":0.872,"15min":0.872},"last":0.71399999999999997}]},{"osd":5,"last update":"Thu Mar 21 12:33:19 2024","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":0.88900000000000001,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.80600000000000005},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"min":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"max":{"1min":0.90700000000000003,"5min":0.90700000000000003,"15min":0.90700000000000003},"last":0.81799999999999995}]},{"osd":6,"last update":"Thu Mar 21 12:33:45 2024","interfaces":[{"interface":"back","average":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"min":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"max":{"1min":0.89500000000000002,"5min":0.89500000000000002,"15min":0.89500000000000002},"last":0.755},{"interface":"front","average":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"min":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"max":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.70499999999999996}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.74399999999999999}]}]},{"osd":2,"up_from":19,"seq":81604378661,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6244,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5632,"kb_avail":93738908,"statfs":{"total":95995035648,"available":95988641792,"internally_reserved":0,"allocated":618496,"data_stored":248214,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5767168},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Mar 21 12:33:11 2024","interfaces":[{"interface":"back","average":{"1min":0.41499999999999998,"5min":0.34599999999999997,"15min":0.33400000000000002},"min":{"1min":0.27500000000000002,"5min":0.218,"15min":0.218},"max":{"1min":0.754,"5min":0.754,"15min":0.754},"last":0.33800000000000002},{"interface":"front","average":{"1min":0.437,"5min":0.36099999999999999,"15min":0.34799999999999998},"min":{"1min":0.309,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"last":0.378}]},{"osd":1,"last update":"Thu Mar 21 12:33:11 2024","interfaces":[{"interface":"back","average":{"1min":0.39300000000000002,"5min":0.36299999999999999,"15min":0.35799999999999998},"min":{"1min":0.29099999999999998,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.51400000000000001,"5min":0.54100000000000004,"15min":0.54100000000000004},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.443,"5min":0.38,"15min":0.37},"min":{"1min":0.34100000000000003,"5min":0.25,"15min":0.25},"max":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"last":0.40799999999999997}]},{"osd":3,"last update":"Thu Mar 21 12:33:37 2024","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.46899999999999997,"15min":0.46100000000000002},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":1.8280000000000001,"5min":1.8280000000000001,"15min":1.8280000000000001},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.48499999999999999,"5min":0.439,"15min":0.43099999999999999},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":1.7230000000000001,"5min":1.7230000000000001,"15min":1.7230000000000001},"last":0.42999999999999999}]},{"osd":4,"last update":"Thu Mar 21 12:33:56 2024","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.56499999999999995,"15min":0.56399999999999995},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.63100000000000001},{"interface":"front","average":{"1min":0.622,"5min":0.59699999999999998,"15min":0.59299999999999997},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.91600000000000004,"5min":0.91600000000000004,"15min":0.91600000000000004},"last":0.55700000000000005}]},{"osd":5,"last update":"Thu Mar 21 12:33:21 2024","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.53100000000000003}]},{"osd":6,"last update":"Thu Mar 21 12:33:45 2024","interfaces":[{"interface":"back","average":{"1min":1.512,"5min":1.512,"15min":1.512},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":16.896000000000001,"5min":16.896000000000001,"15min":16.896000000000001},"last":0.52300000000000002},{"interface":"front","average":{"1min":1.508,"5min":1.508,"15min":1.508},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":16.914999999999999,"5min":16.914999999999999,"15min":16.914999999999999},"last":0.65200000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56399999999999995}]}]},{"osd":3,"up_from":27,"seq":115964117024,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5732,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739420,"statfs":{"total":95995035648,"available":95989166080,"internally_reserved":0,"allocated":618496,"data_stored":248214,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Mar 21 12:33:34 2024","interfaces":[{"interface":"back","average":{"1min":0.40300000000000002,"5min":0.371,"15min":0.36599999999999999},"min":{"1min":0.25700000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":0.66600000000000004,"5min":0.66600000000000004,"15min":0.66600000000000004},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.41799999999999998,"5min":0.40999999999999998,"15min":0.40799999999999997},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":0.625,"5min":0.625,"15min":0.625},"last":0.34599999999999997}]},{"osd":1,"last update":"Thu Mar 21 12:33:34 2024","interfaces":[{"interface":"back","average":{"1min":0.42299999999999999,"5min":0.40300000000000002,"15min":0.39900000000000002},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"last":0.40000000000000002},{"interface":"front","average":{"1min":0.41799999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"min":{"1min":0.28899999999999998,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"last":0.32400000000000001}]},{"osd":2,"last update":"Thu Mar 21 12:33:34 2024","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.438,"15min":0.434},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.44900000000000001,"5min":0.42499999999999999,"15min":0.42099999999999999},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.438}]},{"osd":4,"last update":"Thu Mar 21 12:33:58 2024","interfaces":[{"interface":"back","average":{"1min":0.60899999999999999,"5min":0.56100000000000005,"15min":0.55300000000000005},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.54700000000000004,"15min":0.54200000000000004},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.53100000000000003}]},{"osd":5,"last update":"Thu Mar 21 12:33:21 2024","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.88300000000000001,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.47999999999999998}]},{"osd":6,"last update":"Thu Mar 21 12:33:45 2024","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"min":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"max":{"1min":0.92400000000000004,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":0.79700000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.49099999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.503}]}]},{"osd":4,"up_from":32,"seq":137438953501,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5732,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739420,"statfs":{"total":95995035648,"available":95989166080,"internally_reserved":0,"allocated":618496,"data_stored":248214,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Mar 21 12:34:00 2024","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.57799999999999996,"15min":0.58299999999999996},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.71899999999999997,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.71199999999999997},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.56999999999999995,"15min":0.57399999999999995},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.70299999999999996,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.55700000000000005}]},{"osd":1,"last update":"Thu Mar 21 12:34:00 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.76200000000000001,"15min":0.80000000000000004},"min":{"1min":0.40500000000000003,"5min":0.378,"15min":0.378},"max":{"1min":0.751,"5min":5.351,"15min":5.351},"last":0.69599999999999995},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.57899999999999996,"15min":0.58099999999999996},"min":{"1min":0.372,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.74099999999999999,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.56999999999999995}]},{"osd":2,"last update":"Thu Mar 21 12:34:00 2024","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.56599999999999995,"15min":0.56399999999999995},"min":{"1min":0.41999999999999998,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.77400000000000002,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.77400000000000002},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.58399999999999996,"15min":0.58799999999999997},"min":{"1min":0.40699999999999997,"5min":0.379,"15min":0.379},"max":{"1min":0.76800000000000002,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.60999999999999999}]},{"osd":3,"last update":"Thu Mar 21 12:34:00 2024","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.56599999999999995,"15min":0.56699999999999995},"min":{"1min":0.379,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.68100000000000005},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.58899999999999997,"15min":0.59299999999999997},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.76200000000000001,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.65100000000000002}]},{"osd":5,"last update":"Thu Mar 21 12:33:20 2024","interfaces":[{"interface":"back","average":{"1min":0.374,"5min":0.374,"15min":0.374},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"last":0.67400000000000004},{"interface":"front","average":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.749}]},{"osd":6,"last update":"Thu Mar 21 12:33:45 2024","interfaces":[{"interface":"back","average":{"1min":0.48999999999999999,"5min":0.48999999999999999,"15min":0.48999999999999999},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.62},{"interface":"front","average":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"last":0.59499999999999997}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.73899999999999999}]}]},{"osd":5,"up_from":40,"seq":171798691864,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5668,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739484,"statfs":{"total":95995035648,"available":95989231616,"internally_reserved":0,"allocated":618496,"data_stored":248214,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Mar 21 12:33:19 2024","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.73699999999999999,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.56899999999999995}]},{"osd":1,"last update":"Thu Mar 21 12:33:19 2024","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.59699999999999998},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.70099999999999996,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.52800000000000002}]},{"osd":2,"last update":"Thu Mar 21 12:33:19 2024","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.58399999999999996},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.505}]},{"osd":3,"last update":"Thu Mar 21 12:33:19 2024","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.85599999999999998,"5min":0.85599999999999998,"15min":0.85599999999999998},"last":0.51900000000000002},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.54300000000000004}]},{"osd":4,"last update":"Thu Mar 21 12:33:19 2024","interfaces":[{"interface":"back","average":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.48099999999999998}]},{"osd":6,"last update":"Thu Mar 21 12:33:44 2024","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":0.48799999999999999,"15min":0.48799999999999999},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.38200000000000001},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.41799999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-21T12:34:01.553 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -- ceph pg dump --format=json 2024-03-21T12:34:01.775 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/mon.smithi022/config 2024-03-21T12:34:03.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:02 smithi022 ceph-mon[24777]: pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-21T12:34:03.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:02 smithi022 ceph-mon[24777]: from='client.14448 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-21T12:34:03.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:02 smithi189 ceph-mon[27080]: pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-21T12:34:03.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:02 smithi189 ceph-mon[27080]: from='client.14448 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-21T12:34:03.490 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-03-21T12:34:03.492 INFO:teuthology.orchestra.run.smithi022.stderr:dumped all 2024-03-21T12:34:04.131 INFO:teuthology.orchestra.run.smithi022.stdout:{"pg_ready":true,"pg_map":{"version":196,"stamp":"2024-03-21T12:34:02.856358+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48880,"kb_used_data":6384,"kb_used_omap":0,"kb_used_meta":42432,"kb_avail":749912336,"statfs":{"total":767960285184,"available":767910232064,"internally_reserved":0,"allocated":6537216,"data_stored":3571908,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43450368},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001800"},"pg_stats":[{"pgid":"1.0","version":"23'117","reported_seq":226,"reported_epoch":60,"state":"active+clean","last_fresh":"2024-03-21T12:33:11.938599+0000","last_change":"2024-03-21T12:33:11.938599+0000","last_active":"2024-03-21T12:33:11.938599+0000","last_peered":"2024-03-21T12:33:11.938599+0000","last_clean":"2024-03-21T12:33:11.938599+0000","last_became_active":"2024-03-21T12:33:11.938228+0000","last_became_peered":"2024-03-21T12:33:11.938228+0000","last_unstale":"2024-03-21T12:33:11.938599+0000","last_undegraded":"2024-03-21T12:33:11.938599+0000","last_fullsized":"2024-03-21T12:33:11.938599+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":21,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-21T12:31:09.463044+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-21T12:31:09.463044+0000","last_clean_scrub_stamp":"2024-03-21T12:31:09.463044+0000","objects_scrubbed":0,"log_size":117,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-22T13:40:33.828750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3727360,"data_stored":3702496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":56,"seq":240518168590,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6060,"kb_used_data":1124,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739092,"statfs":{"total":95995035648,"available":95988830208,"internally_reserved":0,"allocated":1150976,"data_stored":777142,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.52800000000000002}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.59299999999999997}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51800000000000002}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.46800000000000003}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.33800000000000002}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.48099999999999998}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54000000000000004}]}]},{"osd":6,"up_from":48,"seq":206158430226,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6180,"kb_used_data":1116,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93738972,"statfs":{"total":95995035648,"available":95988707328,"internally_reserved":0,"allocated":1142784,"data_stored":776554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Mar 21 12:33:46 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.68899999999999995,"5min":0.68899999999999995,"15min":0.68899999999999995},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.51600000000000001}]},{"osd":1,"last update":"Thu Mar 21 12:33:46 2024","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.81399999999999995,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.41199999999999998}]},{"osd":2,"last update":"Thu Mar 21 12:33:46 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.35999999999999999},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.439}]},{"osd":3,"last update":"Thu Mar 21 12:33:46 2024","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.78100000000000003,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"last":0.47199999999999998}]},{"osd":4,"last update":"Thu Mar 21 12:33:46 2024","interfaces":[{"interface":"back","average":{"1min":0.432,"5min":0.432,"15min":0.432},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.375}]},{"osd":5,"last update":"Thu Mar 21 12:33:46 2024","interfaces":[{"interface":"back","average":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":0.39900000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45200000000000001}]}]},{"osd":1,"up_from":14,"seq":60129542187,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6372,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738780,"statfs":{"total":95995035648,"available":95988510720,"internally_reserved":0,"allocated":618496,"data_stored":248214,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Mar 21 12:33:40 2024","interfaces":[{"interface":"back","average":{"1min":0.45200000000000001,"5min":0.36499999999999999,"15min":0.34200000000000003},"min":{"1min":0.25700000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":1.661,"5min":1.661,"15min":1.661},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.40100000000000002,"5min":0.34599999999999997,"15min":0.32500000000000001},"min":{"1min":0.27400000000000002,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.38700000000000001}]},{"osd":2,"last update":"Thu Mar 21 12:33:14 2024","interfaces":[{"interface":"back","average":{"1min":0.39400000000000002,"5min":0.38400000000000001,"15min":0.38200000000000001},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.53200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.39400000000000002,"5min":0.378,"15min":0.375},"min":{"1min":0.28699999999999998,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.54200000000000004,"5min":0.64400000000000002,"15min":0.64400000000000002},"last":0.41399999999999998}]},{"osd":3,"last update":"Thu Mar 21 12:33:40 2024","interfaces":[{"interface":"back","average":{"1min":0.495,"5min":0.45100000000000001,"15min":0.443},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":1.7370000000000001,"5min":1.7370000000000001,"15min":1.7370000000000001},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.5,"5min":0.45600000000000002,"15min":0.44900000000000001},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":1.595,"5min":1.595,"15min":1.595},"last":0.47699999999999998}]},{"osd":4,"last update":"Thu Mar 21 12:33:59 2024","interfaces":[{"interface":"back","average":{"1min":0.83399999999999996,"5min":0.56200000000000006,"15min":0.51700000000000002},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":5.2409999999999997,"5min":5.2409999999999997,"15min":5.2409999999999997},"last":0.49299999999999999},{"interface":"front","average":{"1min":0.65800000000000003,"5min":0.55400000000000005,"15min":0.53700000000000003},"min":{"1min":0.441,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":1.7629999999999999,"5min":1.7629999999999999,"15min":1.7629999999999999},"last":0.54800000000000004}]},{"osd":5,"last update":"Thu Mar 21 12:33:20 2024","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.504},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.53300000000000003}]},{"osd":6,"last update":"Thu Mar 21 12:33:45 2024","interfaces":[{"interface":"back","average":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":1.877,"5min":1.877,"15min":1.877},"last":0.65000000000000002},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.56799999999999995}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.59999999999999998}]}]},{"osd":0,"up_from":9,"seq":38654705714,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6892,"kb_used_data":1124,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738260,"statfs":{"total":95995035648,"available":95987978240,"internally_reserved":0,"allocated":1150976,"data_stored":777142,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Mar 21 12:33:43 2024","interfaces":[{"interface":"back","average":{"1min":0.42299999999999999,"5min":0.34499999999999997,"15min":0.317},"min":{"1min":0.252,"5min":0.214,"15min":0.214},"max":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.44900000000000001,"5min":0.36599999999999999,"15min":0.33300000000000002},"min":{"1min":0.25900000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.54600000000000004}]},{"osd":2,"last update":"Thu Mar 21 12:33:13 2024","interfaces":[{"interface":"back","average":{"1min":0.42199999999999999,"5min":0.40200000000000002,"15min":0.39800000000000002},"min":{"1min":0.27300000000000002,"5min":0.25,"15min":0.25},"max":{"1min":0.61899999999999999,"5min":0.89400000000000002,"15min":0.89400000000000002},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.44700000000000001,"5min":0.39600000000000002,"15min":0.38800000000000001},"min":{"1min":0.34200000000000003,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"last":0.48299999999999998}]},{"osd":3,"last update":"Thu Mar 21 12:33:33 2024","interfaces":[{"interface":"back","average":{"1min":0.42899999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"min":{"1min":0.26600000000000001,"5min":0.253,"15min":0.253},"max":{"1min":0.55600000000000005,"5min":0.63600000000000001,"15min":0.63600000000000001},"last":0.432},{"interface":"front","average":{"1min":0.501,"5min":0.46899999999999997,"15min":0.46300000000000002},"min":{"1min":0.33500000000000002,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.52900000000000003}]},{"osd":4,"last update":"Thu Mar 21 12:34:00 2024","interfaces":[{"interface":"back","average":{"1min":0.64300000000000002,"5min":0.60999999999999999,"15min":0.60399999999999998},"min":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":0.81999999999999995,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.72599999999999998},{"interface":"front","average":{"1min":0.66500000000000004,"5min":0.627,"15min":0.621},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.872,"5min":0.872,"15min":0.872},"last":0.71399999999999997}]},{"osd":5,"last update":"Thu Mar 21 12:33:19 2024","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":0.88900000000000001,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.80600000000000005},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"min":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"max":{"1min":0.90700000000000003,"5min":0.90700000000000003,"15min":0.90700000000000003},"last":0.81799999999999995}]},{"osd":6,"last update":"Thu Mar 21 12:33:45 2024","interfaces":[{"interface":"back","average":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"min":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"max":{"1min":0.89500000000000002,"5min":0.89500000000000002,"15min":0.89500000000000002},"last":0.755},{"interface":"front","average":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"min":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"max":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.70499999999999996}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.74399999999999999}]}]},{"osd":2,"up_from":19,"seq":81604378661,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6244,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5632,"kb_avail":93738908,"statfs":{"total":95995035648,"available":95988641792,"internally_reserved":0,"allocated":618496,"data_stored":248214,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5767168},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Mar 21 12:33:11 2024","interfaces":[{"interface":"back","average":{"1min":0.41499999999999998,"5min":0.34599999999999997,"15min":0.33400000000000002},"min":{"1min":0.27500000000000002,"5min":0.218,"15min":0.218},"max":{"1min":0.754,"5min":0.754,"15min":0.754},"last":0.33800000000000002},{"interface":"front","average":{"1min":0.437,"5min":0.36099999999999999,"15min":0.34799999999999998},"min":{"1min":0.309,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"last":0.378}]},{"osd":1,"last update":"Thu Mar 21 12:33:11 2024","interfaces":[{"interface":"back","average":{"1min":0.39300000000000002,"5min":0.36299999999999999,"15min":0.35799999999999998},"min":{"1min":0.29099999999999998,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.51400000000000001,"5min":0.54100000000000004,"15min":0.54100000000000004},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.443,"5min":0.38,"15min":0.37},"min":{"1min":0.34100000000000003,"5min":0.25,"15min":0.25},"max":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"last":0.40799999999999997}]},{"osd":3,"last update":"Thu Mar 21 12:33:37 2024","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.46899999999999997,"15min":0.46100000000000002},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":1.8280000000000001,"5min":1.8280000000000001,"15min":1.8280000000000001},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.48499999999999999,"5min":0.439,"15min":0.43099999999999999},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":1.7230000000000001,"5min":1.7230000000000001,"15min":1.7230000000000001},"last":0.42999999999999999}]},{"osd":4,"last update":"Thu Mar 21 12:33:56 2024","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.56499999999999995,"15min":0.56399999999999995},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.63100000000000001},{"interface":"front","average":{"1min":0.622,"5min":0.59699999999999998,"15min":0.59299999999999997},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.91600000000000004,"5min":0.91600000000000004,"15min":0.91600000000000004},"last":0.55700000000000005}]},{"osd":5,"last update":"Thu Mar 21 12:33:21 2024","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.53100000000000003}]},{"osd":6,"last update":"Thu Mar 21 12:33:45 2024","interfaces":[{"interface":"back","average":{"1min":1.512,"5min":1.512,"15min":1.512},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":16.896000000000001,"5min":16.896000000000001,"15min":16.896000000000001},"last":0.52300000000000002},{"interface":"front","average":{"1min":1.508,"5min":1.508,"15min":1.508},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":16.914999999999999,"5min":16.914999999999999,"15min":16.914999999999999},"last":0.65200000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56399999999999995}]}]},{"osd":3,"up_from":27,"seq":115964117024,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5732,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739420,"statfs":{"total":95995035648,"available":95989166080,"internally_reserved":0,"allocated":618496,"data_stored":248214,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Mar 21 12:33:34 2024","interfaces":[{"interface":"back","average":{"1min":0.40300000000000002,"5min":0.371,"15min":0.36599999999999999},"min":{"1min":0.25700000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":0.66600000000000004,"5min":0.66600000000000004,"15min":0.66600000000000004},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.41799999999999998,"5min":0.40999999999999998,"15min":0.40799999999999997},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":0.625,"5min":0.625,"15min":0.625},"last":0.34599999999999997}]},{"osd":1,"last update":"Thu Mar 21 12:33:34 2024","interfaces":[{"interface":"back","average":{"1min":0.42299999999999999,"5min":0.40300000000000002,"15min":0.39900000000000002},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"last":0.40000000000000002},{"interface":"front","average":{"1min":0.41799999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"min":{"1min":0.28899999999999998,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"last":0.32400000000000001}]},{"osd":2,"last update":"Thu Mar 21 12:33:34 2024","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.438,"15min":0.434},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.44900000000000001,"5min":0.42499999999999999,"15min":0.42099999999999999},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.438}]},{"osd":4,"last update":"Thu Mar 21 12:33:58 2024","interfaces":[{"interface":"back","average":{"1min":0.60899999999999999,"5min":0.56100000000000005,"15min":0.55300000000000005},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.54700000000000004,"15min":0.54200000000000004},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.53100000000000003}]},{"osd":5,"last update":"Thu Mar 21 12:33:21 2024","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.88300000000000001,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.47999999999999998}]},{"osd":6,"last update":"Thu Mar 21 12:33:45 2024","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"min":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"max":{"1min":0.92400000000000004,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":0.79700000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.49099999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.503}]}]},{"osd":4,"up_from":32,"seq":137438953501,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5732,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739420,"statfs":{"total":95995035648,"available":95989166080,"internally_reserved":0,"allocated":618496,"data_stored":248214,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Mar 21 12:34:00 2024","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.57799999999999996,"15min":0.58299999999999996},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.71899999999999997,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.71199999999999997},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.56999999999999995,"15min":0.57399999999999995},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.70299999999999996,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.55700000000000005}]},{"osd":1,"last update":"Thu Mar 21 12:34:00 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.76200000000000001,"15min":0.80000000000000004},"min":{"1min":0.40500000000000003,"5min":0.378,"15min":0.378},"max":{"1min":0.751,"5min":5.351,"15min":5.351},"last":0.69599999999999995},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.57899999999999996,"15min":0.58099999999999996},"min":{"1min":0.372,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.74099999999999999,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.56999999999999995}]},{"osd":2,"last update":"Thu Mar 21 12:34:00 2024","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.56599999999999995,"15min":0.56399999999999995},"min":{"1min":0.41999999999999998,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.77400000000000002,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.77400000000000002},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.58399999999999996,"15min":0.58799999999999997},"min":{"1min":0.40699999999999997,"5min":0.379,"15min":0.379},"max":{"1min":0.76800000000000002,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.60999999999999999}]},{"osd":3,"last update":"Thu Mar 21 12:34:00 2024","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.56599999999999995,"15min":0.56699999999999995},"min":{"1min":0.379,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.68100000000000005},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.58899999999999997,"15min":0.59299999999999997},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.76200000000000001,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.65100000000000002}]},{"osd":5,"last update":"Thu Mar 21 12:33:20 2024","interfaces":[{"interface":"back","average":{"1min":0.374,"5min":0.374,"15min":0.374},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"last":0.67400000000000004},{"interface":"front","average":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.749}]},{"osd":6,"last update":"Thu Mar 21 12:33:45 2024","interfaces":[{"interface":"back","average":{"1min":0.48999999999999999,"5min":0.48999999999999999,"15min":0.48999999999999999},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.62},{"interface":"front","average":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"last":0.59499999999999997}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.73899999999999999}]}]},{"osd":5,"up_from":40,"seq":171798691864,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5668,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739484,"statfs":{"total":95995035648,"available":95989231616,"internally_reserved":0,"allocated":618496,"data_stored":248214,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Mar 21 12:33:19 2024","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.73699999999999999,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.56899999999999995}]},{"osd":1,"last update":"Thu Mar 21 12:33:19 2024","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.59699999999999998},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.70099999999999996,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.52800000000000002}]},{"osd":2,"last update":"Thu Mar 21 12:33:19 2024","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.58399999999999996},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.505}]},{"osd":3,"last update":"Thu Mar 21 12:33:19 2024","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.85599999999999998,"5min":0.85599999999999998,"15min":0.85599999999999998},"last":0.51900000000000002},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.54300000000000004}]},{"osd":4,"last update":"Thu Mar 21 12:33:19 2024","interfaces":[{"interface":"back","average":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.48099999999999998}]},{"osd":6,"last update":"Thu Mar 21 12:33:44 2024","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":0.48799999999999999,"15min":0.48799999999999999},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.38200000000000001},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.41799999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-21T12:34:04.132 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-03-21T12:34:04.132 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-03-21T12:34:04.132 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-03-21T12:34:04.133 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -- ceph health --format=json 2024-03-21T12:34:04.354 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/mon.smithi022/config 2024-03-21T12:34:05.045 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:04 smithi022 ceph-mon[24777]: pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-21T12:34:05.045 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:04 smithi022 ceph-mon[24777]: from='client.14452 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-21T12:34:05.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:04 smithi189 ceph-mon[27080]: pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-21T12:34:05.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:04 smithi189 ceph-mon[27080]: from='client.14452 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-21T12:34:06.050 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-03-21T12:34:06.050 INFO:teuthology.orchestra.run.smithi022.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-03-21T12:34:06.631 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-03-21T12:34:06.631 INFO:tasks.cephadm:Setup complete, yielding 2024-03-21T12:34:06.631 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-03-21T12:34:06.641 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi022.front.sepia.ceph.com 2024-03-21T12:34:06.641 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -- bash -c 'ceph orch status' 2024-03-21T12:34:06.955 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:06 smithi022 ceph-mon[24777]: pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-21T12:34:06.955 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:06 smithi022 ceph-mon[24777]: from='client.? 172.21.15.22:0/1719846757' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-21T12:34:07.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:06 smithi189 ceph-mon[27080]: pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-21T12:34:07.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:06 smithi189 ceph-mon[27080]: from='client.? 172.21.15.22:0/1719846757' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-21T12:34:07.918 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:07 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi022.fektzs/trash_purge_schedule"}]: dispatch 2024-03-21T12:34:07.918 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:07 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi022.fektzs/mirror_snapshot_schedule"}]: dispatch 2024-03-21T12:34:08.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:07 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi022.fektzs/trash_purge_schedule"}]: dispatch 2024-03-21T12:34:08.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:07 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi022.fektzs/mirror_snapshot_schedule"}]: dispatch 2024-03-21T12:34:08.575 INFO:teuthology.orchestra.run.smithi022.stdout:Backend: cephadm 2024-03-21T12:34:08.575 INFO:teuthology.orchestra.run.smithi022.stdout:Available: Yes 2024-03-21T12:34:08.576 INFO:teuthology.orchestra.run.smithi022.stdout:Paused: No 2024-03-21T12:34:09.149 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:08 smithi022 ceph-mon[24777]: pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-21T12:34:09.174 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -- bash -c 'ceph orch ps' 2024-03-21T12:34:09.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:08 smithi189 ceph-mon[27080]: pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-21T12:34:10.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:09 smithi022 ceph-mon[24777]: from='client.14460 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:34:10.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:09 smithi189 ceph-mon[27080]: from='client.14460 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:34:11.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:10 smithi022 ceph-mon[24777]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-21T12:34:11.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:10 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:34:11.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:10 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:34:11.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:10 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:34:11.197 INFO:teuthology.orchestra.run.smithi022.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-21T12:34:11.197 INFO:teuthology.orchestra.run.smithi022.stdout:alertmanager.smithi022 smithi022 *:9093,9094 running (4m) 2m ago 5m 14.2M - ba2b418f427c 8fbe5dbdce93 2024-03-21T12:34:11.197 INFO:teuthology.orchestra.run.smithi022.stdout:crash.smithi022 smithi022 running (5m) 2m ago 5m 6405k - 17.2.0 e1d6a67b021e 47deb70fd250 2024-03-21T12:34:11.197 INFO:teuthology.orchestra.run.smithi022.stdout:crash.smithi189 smithi189 running (4m) 64s ago 4m 6405k - 17.2.0 e1d6a67b021e 9a2e0e9b1809 2024-03-21T12:34:11.197 INFO:teuthology.orchestra.run.smithi022.stdout:grafana.smithi022 smithi022 *:3000 running (3m) 2m ago 5m 45.2M - 8.3.5 dad864ee21e9 3c9ff131fc6b 2024-03-21T12:34:11.197 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.smithi022.fektzs smithi022 *:9283 running (6m) 2m ago 6m 448M - 17.2.0 e1d6a67b021e ca06c9c2fb41 2024-03-21T12:34:11.197 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.smithi189.cnxzvu smithi189 *:8443,9283 running (4m) 64s ago 4m 412M - 17.2.0 e1d6a67b021e 5ea3c42ad934 2024-03-21T12:34:11.198 INFO:teuthology.orchestra.run.smithi022.stdout:mon.smithi022 smithi022 running (6m) 2m ago 7m 42.7M 2048M 17.2.0 e1d6a67b021e 0d248775e0e5 2024-03-21T12:34:11.198 INFO:teuthology.orchestra.run.smithi022.stdout:mon.smithi189 smithi189 running (4m) 64s ago 4m 39.7M 2048M 17.2.0 e1d6a67b021e 82da5f3cd49a 2024-03-21T12:34:11.198 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.smithi022 smithi022 *:9100 running (5m) 2m ago 5m 20.4M - 1dbe0e931976 cea183555968 2024-03-21T12:34:11.198 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.smithi189 smithi189 *:9100 running (4m) 64s ago 4m 19.5M - 1dbe0e931976 5db95224aff9 2024-03-21T12:34:11.198 INFO:teuthology.orchestra.run.smithi022.stdout:osd.0 smithi022 running (4m) 2m ago 4m 33.0M 3217M 17.2.0 e1d6a67b021e 54e6631bd10f 2024-03-21T12:34:11.198 INFO:teuthology.orchestra.run.smithi022.stdout:osd.1 smithi022 running (3m) 2m ago 3m 34.4M 3217M 17.2.0 e1d6a67b021e 68165c5d09cb 2024-03-21T12:34:11.198 INFO:teuthology.orchestra.run.smithi022.stdout:osd.2 smithi022 running (3m) 2m ago 3m 31.9M 3217M 17.2.0 e1d6a67b021e fbec9bef8433 2024-03-21T12:34:11.198 INFO:teuthology.orchestra.run.smithi022.stdout:osd.3 smithi022 running (2m) 2m ago 2m 25.7M 3217M 17.2.0 e1d6a67b021e b1462ffc1b9e 2024-03-21T12:34:11.198 INFO:teuthology.orchestra.run.smithi022.stdout:osd.4 smithi189 running (2m) 64s ago 2m 32.9M 3984M 17.2.0 e1d6a67b021e 1bedc4eb02ed 2024-03-21T12:34:11.198 INFO:teuthology.orchestra.run.smithi022.stdout:osd.5 smithi189 running (118s) 64s ago 117s 32.9M 3984M 17.2.0 e1d6a67b021e 25fbab9786a5 2024-03-21T12:34:11.198 INFO:teuthology.orchestra.run.smithi022.stdout:osd.6 smithi189 running (93s) 64s ago 93s 30.1M 3984M 17.2.0 e1d6a67b021e 0c192fcfd65b 2024-03-21T12:34:11.198 INFO:teuthology.orchestra.run.smithi022.stdout:osd.7 smithi189 running (69s) 64s ago 69s 11.0M 3984M 17.2.0 e1d6a67b021e 3d0723e1f863 2024-03-21T12:34:11.198 INFO:teuthology.orchestra.run.smithi022.stdout:prometheus.smithi022 smithi022 *:9095 running (4m) 2m ago 4m 56.6M - 514e6a882f6e 20a0eef27ec8 2024-03-21T12:34:11.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:10 smithi189 ceph-mon[27080]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-21T12:34:11.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:10 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:34:11.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:10 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:34:11.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:10 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:34:11.818 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -- bash -c 'ceph orch ls' 2024-03-21T12:34:12.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:11 smithi022 ceph-mon[24777]: pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-21T12:34:12.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:11 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:34:12.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:11 smithi022 ceph-mon[24777]: from='client.14464 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:34:12.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:11 smithi189 ceph-mon[27080]: pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-21T12:34:12.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:11 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:34:12.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:11 smithi189 ceph-mon[27080]: from='client.14464 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:34:13.734 INFO:teuthology.orchestra.run.smithi022.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2024-03-21T12:34:13.734 INFO:teuthology.orchestra.run.smithi022.stdout:alertmanager ?:9093,9094 1/1 2m ago 6m count:1 2024-03-21T12:34:13.735 INFO:teuthology.orchestra.run.smithi022.stdout:crash 2/2 2m ago 6m * 2024-03-21T12:34:13.735 INFO:teuthology.orchestra.run.smithi022.stdout:grafana ?:3000 1/1 2m ago 6m count:1 2024-03-21T12:34:13.735 INFO:teuthology.orchestra.run.smithi022.stdout:mgr 2/2 2m ago 6m count:2 2024-03-21T12:34:13.735 INFO:teuthology.orchestra.run.smithi022.stdout:mon 2/2 2m ago 5m smithi022:172.21.15.22=smithi022;smithi189:172.21.15.189=smithi189;count:2 2024-03-21T12:34:13.735 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter ?:9100 2/2 2m ago 6m * 2024-03-21T12:34:13.735 INFO:teuthology.orchestra.run.smithi022.stdout:osd 8 2m ago - 2024-03-21T12:34:13.735 INFO:teuthology.orchestra.run.smithi022.stdout:prometheus ?:9095 1/1 2m ago 6m count:1 2024-03-21T12:34:14.093 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:13 smithi022 ceph-mon[24777]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-21T12:34:14.093 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:13 smithi022 ceph-mon[24777]: from='client.14468 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:34:14.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:13 smithi189 ceph-mon[27080]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-21T12:34:14.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:13 smithi189 ceph-mon[27080]: from='client.14468 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:34:14.420 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -- bash -c 'ceph orch host ls' 2024-03-21T12:34:15.927 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:15 smithi022 ceph-mon[24777]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-21T12:34:16.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:15 smithi189 ceph-mon[27080]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-21T12:34:16.322 INFO:teuthology.orchestra.run.smithi022.stdout:HOST ADDR LABELS STATUS 2024-03-21T12:34:16.322 INFO:teuthology.orchestra.run.smithi022.stdout:smithi022 172.21.15.22 2024-03-21T12:34:16.322 INFO:teuthology.orchestra.run.smithi022.stdout:smithi189 172.21.15.189 2024-03-21T12:34:16.322 INFO:teuthology.orchestra.run.smithi022.stdout:2 hosts in cluster 2024-03-21T12:34:16.874 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -- bash -c 'ceph orch device ls' 2024-03-21T12:34:18.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:17 smithi022 ceph-mon[24777]: from='client.14472 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:34:18.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:17 smithi189 ceph-mon[27080]: from='client.14472 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:34:18.756 INFO:teuthology.orchestra.run.smithi022.stdout:HOST PATH TYPE DEVICE ID SIZE AVAILABLE REJECT REASONS 2024-03-21T12:34:18.757 INFO:teuthology.orchestra.run.smithi022.stdout:smithi022 /dev/nvme0n1 ssd INTEL_SSDPED1D480GA_PHMB747402EC480DGN 480G LVM detected, locked 2024-03-21T12:34:18.757 INFO:teuthology.orchestra.run.smithi022.stdout:smithi189 /dev/nvme0n1 ssd INTEL_SSDPEDMD400G4_CVFT6232004L400BGN 400G LVM detected, locked 2024-03-21T12:34:19.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:18 smithi022 ceph-mon[24777]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-21T12:34:19.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:18 smithi189 ceph-mon[27080]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-21T12:34:19.595 INFO:teuthology.run_tasks:Running task vip.exec... 2024-03-21T12:34:19.605 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi022.front.sepia.ceph.com 2024-03-21T12:34:19.606 DEBUG:teuthology.orchestra.run.smithi022:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'systemctl stop nfs-server' 2024-03-21T12:34:19.636 INFO:teuthology.orchestra.run.smithi022.stderr:+ systemctl stop nfs-server 2024-03-21T12:34:19.644 INFO:tasks.vip:Running commands on role host.b host ubuntu@smithi189.front.sepia.ceph.com 2024-03-21T12:34:19.644 DEBUG:teuthology.orchestra.run.smithi189:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'systemctl stop nfs-server' 2024-03-21T12:34:19.676 INFO:teuthology.orchestra.run.smithi189.stderr:+ systemctl stop nfs-server 2024-03-21T12:34:19.684 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-03-21T12:34:19.693 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi022.front.sepia.ceph.com 2024-03-21T12:34:19.693 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -- bash -c 'ceph fs volume create foofs' 2024-03-21T12:34:20.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:19 smithi189 ceph-mon[27080]: from='client.14476 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:34:20.145 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:19 smithi022 ceph-mon[24777]: from='client.14476 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:34:20.969 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:20 smithi022 ceph-mon[24777]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-21T12:34:21.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:20 smithi189 ceph-mon[27080]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-21T12:34:22.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:22 smithi189 ceph-mon[27080]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-21T12:34:22.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:22 smithi189 ceph-mon[27080]: from='client.14480 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foofs", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:34:22.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:22 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]: dispatch 2024-03-21T12:34:22.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:22 smithi022 ceph-mon[24777]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-21T12:34:22.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:22 smithi022 ceph-mon[24777]: from='client.14480 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foofs", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:34:22.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:22 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]: dispatch 2024-03-21T12:34:23.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:23 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-mon-smithi022[24773]: 2024-03-21T12:34:22.998+0000 7fd45850b700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-03-21T12:34:23.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:23 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]': finished 2024-03-21T12:34:23.345 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:23 smithi022 ceph-mon[24777]: osdmap e61: 8 total, 8 up, 8 in 2024-03-21T12:34:23.345 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:23 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]: dispatch 2024-03-21T12:34:23.345 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:23 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]': finished 2024-03-21T12:34:23.345 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:23 smithi022 ceph-mon[24777]: osdmap e62: 8 total, 8 up, 8 in 2024-03-21T12:34:23.345 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:23 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]: dispatch 2024-03-21T12:34:23.345 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:23 smithi022 ceph-mon[24777]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-03-21T12:34:23.345 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:23 smithi022 ceph-mon[24777]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-03-21T12:34:23.345 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:23 smithi022 ceph-mon[24777]: osdmap e63: 8 total, 8 up, 8 in 2024-03-21T12:34:23.345 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:23 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]': finished 2024-03-21T12:34:23.345 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:23 smithi022 ceph-mon[24777]: fsmap foofs:0 2024-03-21T12:34:23.345 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:23 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:34:23.345 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:23 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:34:23.345 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:23 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:34:23.345 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:23 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:34:23.346 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:23 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:34:23.346 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:23 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:34:23.346 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:23 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi022.bcnxho", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-21T12:34:23.346 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:23 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi022.bcnxho", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-03-21T12:34:23.346 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:23 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:34:23.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:23 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]': finished 2024-03-21T12:34:23.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:23 smithi189 ceph-mon[27080]: osdmap e61: 8 total, 8 up, 8 in 2024-03-21T12:34:23.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:23 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]: dispatch 2024-03-21T12:34:23.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:23 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]': finished 2024-03-21T12:34:23.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:23 smithi189 ceph-mon[27080]: osdmap e62: 8 total, 8 up, 8 in 2024-03-21T12:34:23.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:23 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]: dispatch 2024-03-21T12:34:23.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:23 smithi189 ceph-mon[27080]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-03-21T12:34:23.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:23 smithi189 ceph-mon[27080]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-03-21T12:34:23.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:23 smithi189 ceph-mon[27080]: osdmap e63: 8 total, 8 up, 8 in 2024-03-21T12:34:23.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:23 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]': finished 2024-03-21T12:34:23.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:23 smithi189 ceph-mon[27080]: fsmap foofs:0 2024-03-21T12:34:23.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:23 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:34:23.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:23 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:34:23.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:23 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:34:23.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:23 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:34:23.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:23 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:34:23.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:23 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:34:23.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:23 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi022.bcnxho", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-21T12:34:23.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:23 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi022.bcnxho", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-03-21T12:34:23.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:23 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:34:23.677 INFO:teuthology.run_tasks:Running task cephadm.wait_for_service... 2024-03-21T12:34:23.685 INFO:tasks.cephadm:Waiting for ceph service mds.foofs to start (timeout 300)... 2024-03-21T12:34:23.685 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -- ceph orch ls -f json 2024-03-21T12:34:24.524 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:24 smithi022 ceph-mon[24777]: pgmap v207: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-21T12:34:24.524 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:24 smithi022 ceph-mon[24777]: Saving service mds.foofs spec with placement count:2 2024-03-21T12:34:24.524 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:24 smithi022 ceph-mon[24777]: Deploying daemon mds.foofs.smithi022.bcnxho on smithi022 2024-03-21T12:34:24.524 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:24 smithi022 ceph-mon[24777]: osdmap e64: 8 total, 8 up, 8 in 2024-03-21T12:34:24.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:24 smithi189 ceph-mon[27080]: pgmap v207: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-21T12:34:24.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:24 smithi189 ceph-mon[27080]: Saving service mds.foofs spec with placement count:2 2024-03-21T12:34:24.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:24 smithi189 ceph-mon[27080]: Deploying daemon mds.foofs.smithi022.bcnxho on smithi022 2024-03-21T12:34:24.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:24 smithi189 ceph-mon[27080]: osdmap e64: 8 total, 8 up, 8 in 2024-03-21T12:34:26.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:26 smithi189 ceph-mon[27080]: pgmap v211: 65 pgs: 2 creating+activating, 20 unknown, 43 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-21T12:34:26.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:26 smithi189 ceph-mon[27080]: osdmap e65: 8 total, 8 up, 8 in 2024-03-21T12:34:26.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:26 smithi022 ceph-mon[24777]: pgmap v211: 65 pgs: 2 creating+activating, 20 unknown, 43 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-21T12:34:26.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:26 smithi022 ceph-mon[24777]: osdmap e65: 8 total, 8 up, 8 in 2024-03-21T12:34:26.765 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-03-21T12:34:26.765 INFO:teuthology.orchestra.run.smithi022.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-03-21T12:28:01.307182Z", "last_refresh": "2024-03-21T12:31:41.318804Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-03-21T12:27:55.198852Z", "last_refresh": "2024-03-21T12:31:41.318871Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-03-21T12:27:58.473916Z", "last_refresh": "2024-03-21T12:31:41.318934Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-03-21T12:34:23.020486Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-03-21T12:34:23.013720Z", "running": 0, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-03-21T12:27:53.563690Z", "last_refresh": "2024-03-21T12:31:41.318731Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi022:172.21.15.22=smithi022", "smithi189:172.21.15.189=smithi189"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-03-21T12:28:45.410421Z", "last_refresh": "2024-03-21T12:31:41.318589Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-03-21T12:27:59.661651Z", "last_refresh": "2024-03-21T12:31:41.318996Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-03-21T12:31:41.319120Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-03-21T12:27:56.660150Z", "last_refresh": "2024-03-21T12:31:41.319058Z", "ports": [9095], "running": 1, "size": 1}}] 2024-03-21T12:34:27.285 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:27 smithi022 ceph-mon[24777]: from='client.14484 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-21T12:34:27.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:27 smithi189 ceph-mon[27080]: from='client.14484 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-21T12:34:27.638 INFO:tasks.cephadm:mds.foofs has 0/2 2024-03-21T12:34:28.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:28 smithi022 ceph-mon[24777]: pgmap v213: 65 pgs: 2 creating+activating, 63 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-03-21T12:34:28.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:28 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:34:28.595 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:28 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi189.lluvko", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-21T12:34:28.595 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:28 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi189.lluvko", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-03-21T12:34:28.595 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:28 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:34:28.595 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:28 smithi022 ceph-mon[24777]: Deploying daemon mds.foofs.smithi189.lluvko on smithi189 2024-03-21T12:34:28.595 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:28 smithi022 ceph-mon[24777]: mds.? [v2:172.21.15.22:6834/2456616276,v1:172.21.15.22:6835/2456616276] up:boot 2024-03-21T12:34:28.595 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:28 smithi022 ceph-mon[24777]: daemon mds.foofs.smithi022.bcnxho assigned to filesystem foofs as rank 0 (now has 1 ranks) 2024-03-21T12:34:28.595 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:28 smithi022 ceph-mon[24777]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-03-21T12:34:28.595 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:28 smithi022 ceph-mon[24777]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-03-21T12:34:28.595 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:28 smithi022 ceph-mon[24777]: Cluster is now healthy 2024-03-21T12:34:28.595 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:28 smithi022 ceph-mon[24777]: fsmap foofs:0 1 up:standby 2024-03-21T12:34:28.595 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:28 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi022.bcnxho"}]: dispatch 2024-03-21T12:34:28.595 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:28 smithi022 ceph-mon[24777]: fsmap foofs:1 {0=foofs.smithi022.bcnxho=up:creating} 2024-03-21T12:34:28.595 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:28 smithi022 ceph-mon[24777]: daemon mds.foofs.smithi022.bcnxho is now active in filesystem foofs as rank 0 2024-03-21T12:34:28.639 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -- ceph orch ls -f json 2024-03-21T12:34:28.735 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:28 smithi189 ceph-mon[27080]: pgmap v213: 65 pgs: 2 creating+activating, 63 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-03-21T12:34:28.735 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:28 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:34:28.735 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:28 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi189.lluvko", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-21T12:34:28.735 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:28 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi189.lluvko", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-03-21T12:34:28.735 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:28 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:34:28.735 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:28 smithi189 ceph-mon[27080]: Deploying daemon mds.foofs.smithi189.lluvko on smithi189 2024-03-21T12:34:28.735 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:28 smithi189 ceph-mon[27080]: mds.? [v2:172.21.15.22:6834/2456616276,v1:172.21.15.22:6835/2456616276] up:boot 2024-03-21T12:34:28.735 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:28 smithi189 ceph-mon[27080]: daemon mds.foofs.smithi022.bcnxho assigned to filesystem foofs as rank 0 (now has 1 ranks) 2024-03-21T12:34:28.735 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:28 smithi189 ceph-mon[27080]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-03-21T12:34:28.735 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:28 smithi189 ceph-mon[27080]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-03-21T12:34:28.735 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:28 smithi189 ceph-mon[27080]: Cluster is now healthy 2024-03-21T12:34:28.735 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:28 smithi189 ceph-mon[27080]: fsmap foofs:0 1 up:standby 2024-03-21T12:34:28.735 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:28 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi022.bcnxho"}]: dispatch 2024-03-21T12:34:28.735 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:28 smithi189 ceph-mon[27080]: fsmap foofs:1 {0=foofs.smithi022.bcnxho=up:creating} 2024-03-21T12:34:28.735 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:28 smithi189 ceph-mon[27080]: daemon mds.foofs.smithi022.bcnxho is now active in filesystem foofs as rank 0 2024-03-21T12:34:30.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:30 smithi189 ceph-mon[27080]: pgmap v214: 65 pgs: 65 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 2 op/s 2024-03-21T12:34:30.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:30 smithi189 ceph-mon[27080]: mds.? [v2:172.21.15.22:6834/2456616276,v1:172.21.15.22:6835/2456616276] up:active 2024-03-21T12:34:30.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:30 smithi189 ceph-mon[27080]: fsmap foofs:1 {0=foofs.smithi022.bcnxho=up:active} 2024-03-21T12:34:30.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:30 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:34:30.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:30 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:34:30.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:30 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:34:30.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:30 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:34:30.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:30 smithi022 ceph-mon[24777]: pgmap v214: 65 pgs: 65 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 2 op/s 2024-03-21T12:34:30.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:30 smithi022 ceph-mon[24777]: mds.? [v2:172.21.15.22:6834/2456616276,v1:172.21.15.22:6835/2456616276] up:active 2024-03-21T12:34:30.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:30 smithi022 ceph-mon[24777]: fsmap foofs:1 {0=foofs.smithi022.bcnxho=up:active} 2024-03-21T12:34:30.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:30 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:34:30.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:30 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:34:30.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:30 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:34:30.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:30 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:34:30.812 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-03-21T12:34:30.813 INFO:teuthology.orchestra.run.smithi022.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-03-21T12:28:01.307182Z", "last_refresh": "2024-03-21T12:31:41.318804Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-03-21T12:27:55.198852Z", "last_refresh": "2024-03-21T12:31:41.318871Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-03-21T12:27:58.473916Z", "last_refresh": "2024-03-21T12:31:41.318934Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-03-21T12:34:23.020486Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-03-21T12:34:23.013720Z", "running": 0, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-03-21T12:27:53.563690Z", "last_refresh": "2024-03-21T12:31:41.318731Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi022:172.21.15.22=smithi022", "smithi189:172.21.15.189=smithi189"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-03-21T12:28:45.410421Z", "last_refresh": "2024-03-21T12:31:41.318589Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-03-21T12:27:59.661651Z", "last_refresh": "2024-03-21T12:31:41.318996Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-03-21T12:31:41.319120Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-03-21T12:27:56.660150Z", "last_refresh": "2024-03-21T12:31:41.319058Z", "ports": [9095], "running": 1, "size": 1}}] 2024-03-21T12:34:31.495 INFO:tasks.cephadm:mds.foofs has 0/2 2024-03-21T12:34:31.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:31 smithi189 ceph-mon[27080]: mds.? [v2:172.21.15.189:6832/2684579462,v1:172.21.15.189:6833/2684579462] up:boot 2024-03-21T12:34:31.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:31 smithi189 ceph-mon[27080]: fsmap foofs:1 {0=foofs.smithi022.bcnxho=up:active} 1 up:standby 2024-03-21T12:34:31.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:31 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi189.lluvko"}]: dispatch 2024-03-21T12:34:31.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:31 smithi189 ceph-mon[27080]: fsmap foofs:1 {0=foofs.smithi022.bcnxho=up:active} 1 up:standby 2024-03-21T12:34:31.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:31 smithi189 ceph-mon[27080]: from='client.14492 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-21T12:34:31.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:31 smithi022 ceph-mon[24777]: mds.? [v2:172.21.15.189:6832/2684579462,v1:172.21.15.189:6833/2684579462] up:boot 2024-03-21T12:34:31.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:31 smithi022 ceph-mon[24777]: fsmap foofs:1 {0=foofs.smithi022.bcnxho=up:active} 1 up:standby 2024-03-21T12:34:31.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:31 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi189.lluvko"}]: dispatch 2024-03-21T12:34:31.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:31 smithi022 ceph-mon[24777]: fsmap foofs:1 {0=foofs.smithi022.bcnxho=up:active} 1 up:standby 2024-03-21T12:34:31.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:31 smithi022 ceph-mon[24777]: from='client.14492 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-21T12:34:32.496 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -- ceph orch ls -f json 2024-03-21T12:34:32.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:32 smithi189 ceph-mon[27080]: pgmap v215: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s wr, 5 op/s 2024-03-21T12:34:32.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:32 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:34:32.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:32 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:34:32.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:32 smithi022 ceph-mon[24777]: pgmap v215: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s wr, 5 op/s 2024-03-21T12:34:32.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:32 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:34:32.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:32 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:34:34.588 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:34 smithi022 ceph-mon[24777]: pgmap v216: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s wr, 4 op/s 2024-03-21T12:34:34.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:34 smithi189 ceph-mon[27080]: pgmap v216: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s wr, 4 op/s 2024-03-21T12:34:35.122 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-03-21T12:34:35.122 INFO:teuthology.orchestra.run.smithi022.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-03-21T12:28:01.307182Z", "last_refresh": "2024-03-21T12:31:41.318804Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-03-21T12:27:55.198852Z", "last_refresh": "2024-03-21T12:31:41.318871Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-03-21T12:27:58.473916Z", "last_refresh": "2024-03-21T12:31:41.318934Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-03-21T12:34:23.020486Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-03-21T12:34:23.013720Z", "last_refresh": "2024-03-21T12:34:31.955413Z", "running": 1, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-03-21T12:27:53.563690Z", "last_refresh": "2024-03-21T12:31:41.318731Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi022:172.21.15.22=smithi022", "smithi189:172.21.15.189=smithi189"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-03-21T12:28:45.410421Z", "last_refresh": "2024-03-21T12:31:41.318589Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-03-21T12:27:59.661651Z", "last_refresh": "2024-03-21T12:31:41.318996Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-03-21T12:31:41.319120Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-03-21T12:27:56.660150Z", "last_refresh": "2024-03-21T12:31:41.319058Z", "ports": [9095], "running": 1, "size": 1}}] 2024-03-21T12:34:36.434 INFO:tasks.cephadm:mds.foofs has 1/2 2024-03-21T12:34:36.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:36 smithi022 ceph-mon[24777]: pgmap v217: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-03-21T12:34:36.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:36 smithi022 ceph-mon[24777]: from='client.14496 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-21T12:34:36.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:36 smithi189 ceph-mon[27080]: pgmap v217: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-03-21T12:34:36.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:36 smithi189 ceph-mon[27080]: from='client.14496 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-21T12:34:37.435 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -- ceph orch ls -f json 2024-03-21T12:34:38.021 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:37 smithi022 ceph-mon[24777]: pgmap v218: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-03-21T12:34:38.022 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:37 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:34:38.022 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:37 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:34:38.022 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:37 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:34:38.022 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:37 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:34:38.022 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:37 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:34:38.022 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:37 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:34:38.022 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:37 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:34:38.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:37 smithi189 ceph-mon[27080]: pgmap v218: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-03-21T12:34:38.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:37 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:34:38.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:37 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:34:38.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:37 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:34:38.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:37 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:34:38.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:37 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:34:38.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:37 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:34:38.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:37 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:34:39.445 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-03-21T12:34:39.445 INFO:teuthology.orchestra.run.smithi022.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-03-21T12:28:01.307182Z", "last_refresh": "2024-03-21T12:34:37.267576Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-03-21T12:27:55.198852Z", "last_refresh": "2024-03-21T12:34:31.954739Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-03-21T12:27:58.473916Z", "last_refresh": "2024-03-21T12:34:37.267707Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-03-21T12:34:23.020486Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-03-21T12:34:23.013720Z", "last_refresh": "2024-03-21T12:34:31.955413Z", "running": 2, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-03-21T12:27:53.563690Z", "last_refresh": "2024-03-21T12:34:31.954900Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi022:172.21.15.22=smithi022", "smithi189:172.21.15.189=smithi189"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-03-21T12:28:45.410421Z", "last_refresh": "2024-03-21T12:34:31.954979Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-03-21T12:27:59.661651Z", "last_refresh": "2024-03-21T12:34:31.955051Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-03-21T12:34:31.955120Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-03-21T12:27:56.660150Z", "last_refresh": "2024-03-21T12:34:37.267832Z", "ports": [9095], "running": 1, "size": 1}}] 2024-03-21T12:34:40.029 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:39 smithi022 ceph-mon[24777]: pgmap v219: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-03-21T12:34:40.029 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:39 smithi022 ceph-mon[24777]: from='client.14500 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-21T12:34:40.030 INFO:tasks.cephadm:mds.foofs has 2/2 2024-03-21T12:34:40.031 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-03-21T12:34:40.041 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi022.front.sepia.ceph.com 2024-03-21T12:34:40.042 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -- bash -c 'ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2' 2024-03-21T12:34:40.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:39 smithi189 ceph-mon[27080]: pgmap v219: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-03-21T12:34:40.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:39 smithi189 ceph-mon[27080]: from='client.14500 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-21T12:34:42.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:41 smithi189 ceph-mon[27080]: pgmap v220: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 2 op/s 2024-03-21T12:34:42.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:41 smithi022 ceph-mon[24777]: pgmap v220: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 2 op/s 2024-03-21T12:34:43.257 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:42 smithi022 ceph-mon[24777]: from='client.14504 -' entity='client.admin' cmd=[{"prefix": "nfs cluster create", "cluster_id": "foo", "placement": "2", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:34:43.257 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:42 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd pool create", "pool": ".nfs"}]: dispatch 2024-03-21T12:34:43.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:42 smithi189 ceph-mon[27080]: from='client.14504 -' entity='client.admin' cmd=[{"prefix": "nfs cluster create", "cluster_id": "foo", "placement": "2", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:34:43.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:42 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd pool create", "pool": ".nfs"}]: dispatch 2024-03-21T12:34:43.949 INFO:teuthology.orchestra.run.smithi022.stdout:NFS Cluster Created Successfully 2024-03-21T12:34:44.164 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:43 smithi022 ceph-mon[24777]: pgmap v221: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-03-21T12:34:44.164 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:43 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "osd pool create", "pool": ".nfs"}]': finished 2024-03-21T12:34:44.164 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:43 smithi022 ceph-mon[24777]: osdmap e66: 8 total, 8 up, 8 in 2024-03-21T12:34:44.164 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:43 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]: dispatch 2024-03-21T12:34:44.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:43 smithi189 ceph-mon[27080]: pgmap v221: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-03-21T12:34:44.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:43 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "osd pool create", "pool": ".nfs"}]': finished 2024-03-21T12:34:44.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:43 smithi189 ceph-mon[27080]: osdmap e66: 8 total, 8 up, 8 in 2024-03-21T12:34:44.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:43 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]: dispatch 2024-03-21T12:34:44.721 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -- bash -c 'ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake' 2024-03-21T12:34:45.000 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:44 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]': finished 2024-03-21T12:34:45.000 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:44 smithi022 ceph-mon[24777]: osdmap e67: 8 total, 8 up, 8 in 2024-03-21T12:34:45.000 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:44 smithi022 ceph-mon[24777]: Saving service nfs.foo spec with placement count:2 2024-03-21T12:34:45.000 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:44 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:34:45.000 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:44 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:34:45.000 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:44 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:34:45.000 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:44 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:34:45.001 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:44 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:34:45.001 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:44 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:34:45.001 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:44 smithi022 ceph-mon[24777]: Creating key for client.nfs.foo.0.0.smithi022.ynhbbq 2024-03-21T12:34:45.001 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:44 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi022.ynhbbq", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-03-21T12:34:45.001 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:44 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi022.ynhbbq", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-03-21T12:34:45.001 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:44 smithi022 ceph-mon[24777]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-03-21T12:34:45.001 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:44 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-03-21T12:34:45.001 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:44 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-03-21T12:34:45.001 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:44 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:34:45.001 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:44 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-03-21T12:34:45.001 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:44 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-03-21T12:34:45.002 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:44 smithi022 ceph-mon[24777]: Rados config object exists: conf-nfs.foo 2024-03-21T12:34:45.002 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:44 smithi022 ceph-mon[24777]: Creating key for client.nfs.foo.0.0.smithi022.ynhbbq-rgw 2024-03-21T12:34:45.002 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:44 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi022.ynhbbq-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-03-21T12:34:45.002 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:44 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi022.ynhbbq-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-03-21T12:34:45.002 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:44 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:34:45.002 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:44 smithi022 ceph-mon[24777]: Deploying daemon nfs.foo.0.0.smithi022.ynhbbq on smithi022 2024-03-21T12:34:45.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:44 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]': finished 2024-03-21T12:34:45.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:44 smithi189 ceph-mon[27080]: osdmap e67: 8 total, 8 up, 8 in 2024-03-21T12:34:45.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:44 smithi189 ceph-mon[27080]: Saving service nfs.foo spec with placement count:2 2024-03-21T12:34:45.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:44 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:34:45.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:44 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:34:45.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:44 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:34:45.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:44 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:34:45.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:44 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:34:45.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:44 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:34:45.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:44 smithi189 ceph-mon[27080]: Creating key for client.nfs.foo.0.0.smithi022.ynhbbq 2024-03-21T12:34:45.287 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:44 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi022.ynhbbq", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-03-21T12:34:45.287 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:44 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi022.ynhbbq", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-03-21T12:34:45.287 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:44 smithi189 ceph-mon[27080]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-03-21T12:34:45.287 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:44 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-03-21T12:34:45.287 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:44 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-03-21T12:34:45.287 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:44 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:34:45.287 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:44 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-03-21T12:34:45.287 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:44 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-03-21T12:34:45.287 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:44 smithi189 ceph-mon[27080]: Rados config object exists: conf-nfs.foo 2024-03-21T12:34:45.287 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:44 smithi189 ceph-mon[27080]: Creating key for client.nfs.foo.0.0.smithi022.ynhbbq-rgw 2024-03-21T12:34:45.287 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:44 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi022.ynhbbq-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-03-21T12:34:45.287 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:44 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi022.ynhbbq-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-03-21T12:34:45.287 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:44 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:34:45.287 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:44 smithi189 ceph-mon[27080]: Deploying daemon nfs.foo.0.0.smithi022.ynhbbq on smithi022 2024-03-21T12:34:46.219 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:45 smithi022 ceph-mon[24777]: pgmap v224: 97 pgs: 8 creating+peering, 17 unknown, 72 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-03-21T12:34:46.219 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:45 smithi022 ceph-mon[24777]: osdmap e68: 8 total, 8 up, 8 in 2024-03-21T12:34:46.219 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:45 smithi022 ceph-mon[24777]: mgrmap e20: smithi022.fektzs(active, since 5m), standbys: smithi189.cnxzvu 2024-03-21T12:34:46.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:45 smithi189 ceph-mon[27080]: pgmap v224: 97 pgs: 8 creating+peering, 17 unknown, 72 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-03-21T12:34:46.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:45 smithi189 ceph-mon[27080]: osdmap e68: 8 total, 8 up, 8 in 2024-03-21T12:34:46.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:45 smithi189 ceph-mon[27080]: mgrmap e20: smithi022.fektzs(active, since 5m), standbys: smithi189.cnxzvu 2024-03-21T12:34:48.165 INFO:teuthology.orchestra.run.smithi022.stderr:Invalid command: Unexpected argument '--clusterid' 2024-03-21T12:34:48.166 INFO:teuthology.orchestra.run.smithi022.stderr:nfs export create cephfs [] [--readonly] [--client_addr ...] [--squash ] : Create a CephFS export 2024-03-21T12:34:48.166 INFO:teuthology.orchestra.run.smithi022.stderr:Error EINVAL: invalid command 2024-03-21T12:34:48.329 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:48 smithi022 ceph-mon[24777]: pgmap v226: 97 pgs: 8 creating+peering, 89 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-03-21T12:34:48.523 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-03-21T12:34:48.523 INFO:teuthology.orchestra.run.smithi022.stdout: "bind": "/fake", 2024-03-21T12:34:48.523 INFO:teuthology.orchestra.run.smithi022.stdout: "fs": "foofs", 2024-03-21T12:34:48.524 INFO:teuthology.orchestra.run.smithi022.stdout: "path": "/", 2024-03-21T12:34:48.524 INFO:teuthology.orchestra.run.smithi022.stdout: "cluster": "foo", 2024-03-21T12:34:48.524 INFO:teuthology.orchestra.run.smithi022.stdout: "mode": "RW" 2024-03-21T12:34:48.524 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-03-21T12:34:48.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:48 smithi189 ceph-mon[27080]: pgmap v226: 97 pgs: 8 creating+peering, 89 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-03-21T12:34:49.206 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -- bash -c 'while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done' 2024-03-21T12:34:49.751 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:49 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:34:49.751 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:49 smithi022 ceph-mon[24777]: Creating key for client.nfs.foo.1.0.smithi189.yzsqml 2024-03-21T12:34:49.751 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:49 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi189.yzsqml", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-03-21T12:34:49.751 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:49 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi189.yzsqml", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-03-21T12:34:49.751 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:49 smithi022 ceph-mon[24777]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-03-21T12:34:49.751 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:49 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-03-21T12:34:49.751 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:49 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-03-21T12:34:49.752 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:49 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:34:49.752 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:49 smithi022 ceph-mon[24777]: from='client.14526 -' entity='client.admin' cmd=[{"prefix": "nfs export create cephfs", "fsname": "foofs", "cluster_id": "foo", "pseudo_path": "/fake", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:34:49.752 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:49 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2024-03-21T12:34:49.752 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:49 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2024-03-21T12:34:49.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:49 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:34:49.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:49 smithi189 ceph-mon[27080]: Creating key for client.nfs.foo.1.0.smithi189.yzsqml 2024-03-21T12:34:49.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:49 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi189.yzsqml", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-03-21T12:34:49.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:49 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi189.yzsqml", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-03-21T12:34:49.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:49 smithi189 ceph-mon[27080]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-03-21T12:34:49.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:49 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-03-21T12:34:49.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:49 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-03-21T12:34:49.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:49 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:34:49.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:49 smithi189 ceph-mon[27080]: from='client.14526 -' entity='client.admin' cmd=[{"prefix": "nfs export create cephfs", "fsname": "foofs", "cluster_id": "foo", "pseudo_path": "/fake", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:34:49.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:49 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2024-03-21T12:34:49.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:49 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2024-03-21T12:34:50.706 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:50 smithi022 ceph-mon[24777]: pgmap v227: 97 pgs: 97 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1.7 KiB/s wr, 3 op/s 2024-03-21T12:34:50.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:50 smithi189 ceph-mon[27080]: pgmap v227: 97 pgs: 97 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1.7 KiB/s wr, 3 op/s 2024-03-21T12:34:51.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:51 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-03-21T12:34:51.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:51 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-03-21T12:34:51.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:51 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi189.yzsqml-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-03-21T12:34:51.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:51 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi189.yzsqml-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-03-21T12:34:51.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:51 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:34:51.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:51 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-03-21T12:34:51.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:51 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-03-21T12:34:51.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:51 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi189.yzsqml-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-03-21T12:34:51.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:51 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi189.yzsqml-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-03-21T12:34:51.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:51 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:34:52.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:52 smithi189 ceph-mon[27080]: pgmap v228: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1.4 KiB/s wr, 3 op/s 2024-03-21T12:34:52.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:52 smithi189 ceph-mon[27080]: from='client.14542 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:34:52.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:52 smithi189 ceph-mon[27080]: Rados config object exists: conf-nfs.foo 2024-03-21T12:34:52.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:52 smithi189 ceph-mon[27080]: Creating key for client.nfs.foo.1.0.smithi189.yzsqml-rgw 2024-03-21T12:34:52.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:52 smithi189 ceph-mon[27080]: Deploying daemon nfs.foo.1.0.smithi189.yzsqml on smithi189 2024-03-21T12:34:52.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:52 smithi022 ceph-mon[24777]: pgmap v228: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1.4 KiB/s wr, 3 op/s 2024-03-21T12:34:52.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:52 smithi022 ceph-mon[24777]: from='client.14542 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:34:52.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:52 smithi022 ceph-mon[24777]: Rados config object exists: conf-nfs.foo 2024-03-21T12:34:52.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:52 smithi022 ceph-mon[24777]: Creating key for client.nfs.foo.1.0.smithi189.yzsqml-rgw 2024-03-21T12:34:52.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:52 smithi022 ceph-mon[24777]: Deploying daemon nfs.foo.1.0.smithi189.yzsqml on smithi189 2024-03-21T12:34:53.794 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:53 smithi189 ceph-mon[27080]: from='client.14556 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:34:53.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:53 smithi022 ceph-mon[24777]: from='client.14556 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:34:54.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:54 smithi189 ceph-mon[27080]: pgmap v229: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.2 KiB/s wr, 2 op/s 2024-03-21T12:34:54.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:54 smithi189 ceph-mon[27080]: from='client.14560 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:34:54.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:54 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:34:54.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:54 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:34:54.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:54 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:34:54.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:54 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:34:54.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:54 smithi022 ceph-mon[24777]: pgmap v229: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.2 KiB/s wr, 2 op/s 2024-03-21T12:34:54.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:54 smithi022 ceph-mon[24777]: from='client.14560 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:34:54.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:54 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:34:54.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:54 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:34:54.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:54 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:34:54.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:54 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:34:56.632 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:56 smithi189 ceph-mon[27080]: pgmap v230: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1.1 KiB/s wr, 2 op/s 2024-03-21T12:34:56.632 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:56 smithi189 ceph-mon[27080]: from='client.14570 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:34:56.843 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:56 smithi022 ceph-mon[24777]: pgmap v230: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1.1 KiB/s wr, 2 op/s 2024-03-21T12:34:56.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:56 smithi022 ceph-mon[24777]: from='client.14570 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:34:58.273 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:57 smithi022 ceph-mon[24777]: from='client.14574 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:34:58.273 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:57 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:34:58.273 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:57 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:34:58.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:57 smithi189 ceph-mon[27080]: from='client.14574 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:34:58.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:57 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:34:58.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:57 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:34:58.988 INFO:teuthology.orchestra.run.smithi022.stdout:nfs.foo ?:2049 2/2 2s ago 15s count:2 2024-03-21T12:34:58.988 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:58 smithi022 ceph-mon[24777]: pgmap v231: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1.6 KiB/s wr, 3 op/s 2024-03-21T12:34:58.988 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:58 smithi022 ceph-mon[24777]: from='client.14578 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:34:58.988 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:58 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:34:58.989 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:58 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:34:58.989 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:58 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:34:58.989 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:58 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:34:58.989 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:58 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:34:58.989 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:58 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:34:58.989 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:58 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:34:58.989 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:34:58 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:34:59.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:58 smithi189 ceph-mon[27080]: pgmap v231: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1.6 KiB/s wr, 3 op/s 2024-03-21T12:34:59.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:58 smithi189 ceph-mon[27080]: from='client.14578 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:34:59.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:58 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:34:59.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:58 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:34:59.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:58 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:34:59.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:58 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:34:59.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:58 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:34:59.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:58 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:34:59.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:58 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:34:59.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:34:58 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:34:59.720 INFO:teuthology.run_tasks:Running task vip.exec... 2024-03-21T12:34:59.733 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi022.front.sepia.ceph.com 2024-03-21T12:34:59.733 DEBUG:teuthology.orchestra.run.smithi022:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'mkdir /mnt/foo' 2024-03-21T12:34:59.761 INFO:teuthology.orchestra.run.smithi022.stderr:+ mkdir /mnt/foo 2024-03-21T12:34:59.777 DEBUG:teuthology.orchestra.run.smithi022:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done' 2024-03-21T12:34:59.843 INFO:teuthology.orchestra.run.smithi022.stderr:++ hostname 2024-03-21T12:34:59.844 INFO:teuthology.orchestra.run.smithi022.stderr:+ mount -t nfs smithi022:/fake /mnt/foo -o sync 2024-03-21T12:35:00.242 DEBUG:teuthology.orchestra.run.smithi022:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'echo test > /mnt/foo/testfile' 2024-03-21T12:35:00.309 INFO:teuthology.orchestra.run.smithi022.stderr:+ echo test 2024-03-21T12:35:00.328 DEBUG:teuthology.orchestra.run.smithi022:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c sync 2024-03-21T12:35:00.393 INFO:teuthology.orchestra.run.smithi022.stderr:+ sync 2024-03-21T12:35:00.634 INFO:teuthology.run_tasks:Running task parallel... 2024-03-21T12:35:00.668 INFO:teuthology.task.parallel:starting parallel... 2024-03-21T12:35:00.668 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-03-21T12:35:00.676 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-03-21T12:35:00.676 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi022.front.sepia.ceph.com 2024-03-21T12:35:00.677 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -e sha1=960fbbe4645c2becc8084669be194c95183c7753 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-03-21T12:35:00.677 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-03-21T12:35:00.677 INFO:teuthology.task.sequential:In sequential, running task exec... 2024-03-21T12:35:00.678 INFO:teuthology.task.exec:Executing custom commands... 2024-03-21T12:35:00.678 INFO:teuthology.task.exec:Running commands on role host.a host ubuntu@smithi022.front.sepia.ceph.com 2024-03-21T12:35:00.678 DEBUG:teuthology.orchestra.run.smithi022:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'cd /mnt/foo && dbench 5 -t 600 || true' 2024-03-21T12:35:00.715 INFO:teuthology.orchestra.run.smithi022.stdout:dbench version 4.00 - Copyright Andrew Tridgell 1999-2004 2024-03-21T12:35:00.715 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-03-21T12:35:00.715 INFO:teuthology.orchestra.run.smithi022.stdout:Running for 600 seconds with load '/usr/share/dbench/client.txt' and minimum warmup 120 secs 2024-03-21T12:35:00.716 INFO:teuthology.orchestra.run.smithi022.stdout:3 of 5 processes prepared for launch 0 sec 2024-03-21T12:35:00.816 INFO:teuthology.orchestra.run.smithi022.stdout:5 of 5 processes prepared for launch 0 sec 2024-03-21T12:35:00.816 INFO:teuthology.orchestra.run.smithi022.stdout:releasing clients 2024-03-21T12:35:00.954 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:35:00 smithi022 ceph-mon[24777]: pgmap v232: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.5 KiB/s rd, 2.2 KiB/s wr, 6 op/s 2024-03-21T12:35:00.954 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:35:00 smithi022 ceph-mon[24777]: from='client.14582 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:35:01.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:35:00 smithi189 ceph-mon[27080]: pgmap v232: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.5 KiB/s rd, 2.2 KiB/s wr, 6 op/s 2024-03-21T12:35:01.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:35:00 smithi189 ceph-mon[27080]: from='client.14582 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:35:01.816 INFO:teuthology.orchestra.run.smithi022.stdout: 5 243 46.44 MB/sec warmup 1 sec latency 23.445 ms 2024-03-21T12:35:02.816 INFO:teuthology.orchestra.run.smithi022.stdout: 5 536 46.38 MB/sec warmup 2 sec latency 15.255 ms 2024-03-21T12:35:02.981 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:35:02 smithi022 ceph-mon[24777]: pgmap v233: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 2.6 KiB/s rd, 1.4 KiB/s wr, 4 op/s 2024-03-21T12:35:03.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:35:02 smithi189 ceph-mon[27080]: pgmap v233: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 2.6 KiB/s rd, 1.4 KiB/s wr, 4 op/s 2024-03-21T12:35:03.816 INFO:teuthology.orchestra.run.smithi022.stdout: 5 1160 38.35 MB/sec warmup 3 sec latency 10.382 ms 2024-03-21T12:35:04.740 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -e sha1=960fbbe4645c2becc8084669be194c95183c7753 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-03-21T12:35:04.816 INFO:teuthology.orchestra.run.smithi022.stdout: 5 2362 34.69 MB/sec warmup 4 sec latency 10.163 ms 2024-03-21T12:35:04.992 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:35:04 smithi022 ceph-mon[24777]: pgmap v234: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 2.6 KiB/s rd, 1.3 KiB/s wr, 4 op/s 2024-03-21T12:35:05.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:35:04 smithi189 ceph-mon[27080]: pgmap v234: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 2.6 KiB/s rd, 1.3 KiB/s wr, 4 op/s 2024-03-21T12:35:05.816 INFO:teuthology.orchestra.run.smithi022.stdout: 5 3432 37.17 MB/sec warmup 5 sec latency 18.402 ms 2024-03-21T12:35:06.816 INFO:teuthology.orchestra.run.smithi022.stdout: 5 3980 33.39 MB/sec warmup 6 sec latency 51.346 ms 2024-03-21T12:35:07.042 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:35:06 smithi022 ceph-mon[24777]: pgmap v235: 97 pgs: 97 active+clean; 71 MiB data, 118 MiB used, 715 GiB / 715 GiB avail; 2.6 KiB/s rd, 6.2 MiB/s wr, 229 op/s 2024-03-21T12:35:07.042 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:35:06 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.8", "id": [7, 2]}]: dispatch 2024-03-21T12:35:07.042 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:35:06 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2024-03-21T12:35:07.042 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:35:06 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2024-03-21T12:35:07.042 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:35:06 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]: dispatch 2024-03-21T12:35:07.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:35:06 smithi189 ceph-mon[27080]: pgmap v235: 97 pgs: 97 active+clean; 71 MiB data, 118 MiB used, 715 GiB / 715 GiB avail; 2.6 KiB/s rd, 6.2 MiB/s wr, 229 op/s 2024-03-21T12:35:07.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:35:06 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.8", "id": [7, 2]}]: dispatch 2024-03-21T12:35:07.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:35:06 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2024-03-21T12:35:07.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:35:06 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2024-03-21T12:35:07.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:35:06 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]: dispatch 2024-03-21T12:35:07.816 INFO:teuthology.orchestra.run.smithi022.stdout: 5 4926 31.40 MB/sec warmup 7 sec latency 22.147 ms 2024-03-21T12:35:08.054 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:35:07 smithi022 ceph-mon[24777]: pgmap v236: 97 pgs: 97 active+clean; 113 MiB data, 184 MiB used, 715 GiB / 715 GiB avail; 2.3 KiB/s rd, 10 MiB/s wr, 470 op/s 2024-03-21T12:35:08.054 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:35:07 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.8", "id": [7, 2]}]': finished 2024-03-21T12:35:08.054 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:35:07 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]': finished 2024-03-21T12:35:08.054 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:35:07 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]': finished 2024-03-21T12:35:08.054 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:35:07 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]': finished 2024-03-21T12:35:08.054 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:35:07 smithi022 ceph-mon[24777]: osdmap e69: 8 total, 8 up, 8 in 2024-03-21T12:35:08.054 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:35:07 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi022.fektzs/trash_purge_schedule"}]: dispatch 2024-03-21T12:35:08.054 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:35:07 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi022.fektzs/mirror_snapshot_schedule"}]: dispatch 2024-03-21T12:35:08.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:35:07 smithi189 ceph-mon[27080]: pgmap v236: 97 pgs: 97 active+clean; 113 MiB data, 184 MiB used, 715 GiB / 715 GiB avail; 2.3 KiB/s rd, 10 MiB/s wr, 470 op/s 2024-03-21T12:35:08.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:35:07 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.8", "id": [7, 2]}]': finished 2024-03-21T12:35:08.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:35:07 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]': finished 2024-03-21T12:35:08.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:35:07 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]': finished 2024-03-21T12:35:08.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:35:07 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]': finished 2024-03-21T12:35:08.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:35:07 smithi189 ceph-mon[27080]: osdmap e69: 8 total, 8 up, 8 in 2024-03-21T12:35:08.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:35:07 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi022.fektzs/trash_purge_schedule"}]: dispatch 2024-03-21T12:35:08.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:35:07 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi022.fektzs/mirror_snapshot_schedule"}]: dispatch 2024-03-21T12:35:08.816 INFO:teuthology.orchestra.run.smithi022.stdout: 5 5913 29.17 MB/sec warmup 8 sec latency 15.144 ms 2024-03-21T12:35:09.060 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:35:08 smithi022 ceph-mon[24777]: osdmap e70: 8 total, 8 up, 8 in 2024-03-21T12:35:09.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:35:08 smithi189 ceph-mon[27080]: osdmap e70: 8 total, 8 up, 8 in 2024-03-21T12:35:09.816 INFO:teuthology.orchestra.run.smithi022.stdout: 5 6720 29.73 MB/sec warmup 9 sec latency 30.596 ms 2024-03-21T12:35:10.075 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:35:09 smithi022 ceph-mon[24777]: pgmap v239: 97 pgs: 97 active+clean; 117 MiB data, 185 MiB used, 715 GiB / 715 GiB avail; 4.9 KiB/s rd, 17 MiB/s wr, 803 op/s 2024-03-21T12:35:10.269 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -e sha1=960fbbe4645c2becc8084669be194c95183c7753 -- bash -c 'ceph config set global log_to_journald false --force' 2024-03-21T12:35:10.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:35:09 smithi189 ceph-mon[27080]: pgmap v239: 97 pgs: 97 active+clean; 117 MiB data, 185 MiB used, 715 GiB / 715 GiB avail; 4.9 KiB/s rd, 17 MiB/s wr, 803 op/s 2024-03-21T12:35:10.816 INFO:teuthology.orchestra.run.smithi022.stdout: 5 7505 29.59 MB/sec warmup 10 sec latency 26.936 ms 2024-03-21T12:35:11.817 INFO:teuthology.orchestra.run.smithi022.stdout: 5 8333 28.11 MB/sec warmup 11 sec latency 20.277 ms 2024-03-21T12:35:12.031 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:35:11 smithi022 ceph-mon[24777]: pgmap v240: 97 pgs: 97 active+clean; 136 MiB data, 290 MiB used, 715 GiB / 715 GiB avail; 4.9 KiB/s rd, 23 MiB/s wr, 1.39k op/s 2024-03-21T12:35:12.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:35:11 smithi189 ceph-mon[27080]: pgmap v240: 97 pgs: 97 active+clean; 136 MiB data, 290 MiB used, 715 GiB / 715 GiB avail; 4.9 KiB/s rd, 23 MiB/s wr, 1.39k op/s 2024-03-21T12:35:12.817 INFO:teuthology.orchestra.run.smithi022.stdout: 5 9403 27.42 MB/sec warmup 12 sec latency 33.627 ms 2024-03-21T12:35:13.817 INFO:teuthology.orchestra.run.smithi022.stdout: 5 10381 28.31 MB/sec warmup 13 sec latency 25.852 ms 2024-03-21T12:35:14.311 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:35:14 smithi022 ceph-mon[24777]: pgmap v241: 97 pgs: 97 active+clean; 136 MiB data, 290 MiB used, 715 GiB / 715 GiB avail; 4.9 KiB/s rd, 14 MiB/s wr, 1.05k op/s 2024-03-21T12:35:14.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:35:14 smithi189 ceph-mon[27080]: pgmap v241: 97 pgs: 97 active+clean; 136 MiB data, 290 MiB used, 715 GiB / 715 GiB avail; 4.9 KiB/s rd, 14 MiB/s wr, 1.05k op/s 2024-03-21T12:35:14.817 INFO:teuthology.orchestra.run.smithi022.stdout: 5 10926 27.88 MB/sec warmup 14 sec latency 25.697 ms 2024-03-21T12:35:15.413 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -e sha1=960fbbe4645c2becc8084669be194c95183c7753 -- bash -c 'ceph mgr module enable nfs --force' 2024-03-21T12:35:15.817 INFO:teuthology.orchestra.run.smithi022.stdout: 5 11429 26.25 MB/sec warmup 15 sec latency 22.016 ms 2024-03-21T12:35:16.329 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:35:16 smithi022 ceph-mon[24777]: pgmap v242: 97 pgs: 97 active+clean; 149 MiB data, 376 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 14 MiB/s wr, 1.34k op/s 2024-03-21T12:35:16.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:35:16 smithi189 ceph-mon[27080]: pgmap v242: 97 pgs: 97 active+clean; 149 MiB data, 376 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 14 MiB/s wr, 1.34k op/s 2024-03-21T12:35:16.817 INFO:teuthology.orchestra.run.smithi022.stdout: 5 12014 25.51 MB/sec warmup 16 sec latency 22.872 ms 2024-03-21T12:35:17.817 INFO:teuthology.orchestra.run.smithi022.stdout: 5 12872 24.87 MB/sec warmup 17 sec latency 36.610 ms 2024-03-21T12:35:18.358 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:35:18 smithi022 ceph-mon[24777]: pgmap v243: 97 pgs: 97 active+clean; 153 MiB data, 429 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 13 MiB/s wr, 1.20k op/s 2024-03-21T12:35:18.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:35:18 smithi189 ceph-mon[27080]: pgmap v243: 97 pgs: 97 active+clean; 153 MiB data, 429 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 13 MiB/s wr, 1.20k op/s 2024-03-21T12:35:18.817 INFO:teuthology.orchestra.run.smithi022.stdout: 5 13414 24.59 MB/sec warmup 18 sec latency 25.661 ms 2024-03-21T12:35:19.231 INFO:teuthology.orchestra.run.smithi022.stderr:module 'nfs' is already enabled 2024-03-21T12:35:19.380 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:35:19 smithi022 ceph-mon[24777]: from='client.? 172.21.15.22:0/2001716470' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-03-21T12:35:19.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:35:19 smithi189 ceph-mon[27080]: from='client.? 172.21.15.22:0/2001716470' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-03-21T12:35:19.817 INFO:teuthology.orchestra.run.smithi022.stdout: 5 14009 24.50 MB/sec warmup 19 sec latency 41.094 ms 2024-03-21T12:35:20.324 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -e sha1=960fbbe4645c2becc8084669be194c95183c7753 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-03-21T12:35:20.385 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:35:20 smithi022 ceph-mon[24777]: pgmap v244: 97 pgs: 97 active+clean; 152 MiB data, 427 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 12 MiB/s wr, 1.12k op/s 2024-03-21T12:35:20.385 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:35:20 smithi022 ceph-mon[24777]: from='client.? 172.21.15.22:0/2001716470' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "nfs", "force": true}]': finished 2024-03-21T12:35:20.386 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:35:20 smithi022 ceph-mon[24777]: mgrmap e21: smithi022.fektzs(active, since 6m), standbys: smithi189.cnxzvu 2024-03-21T12:35:20.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:35:20 smithi189 ceph-mon[27080]: pgmap v244: 97 pgs: 97 active+clean; 152 MiB data, 427 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 12 MiB/s wr, 1.12k op/s 2024-03-21T12:35:20.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:35:20 smithi189 ceph-mon[27080]: from='client.? 172.21.15.22:0/2001716470' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "nfs", "force": true}]': finished 2024-03-21T12:35:20.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:35:20 smithi189 ceph-mon[27080]: mgrmap e21: smithi022.fektzs(active, since 6m), standbys: smithi189.cnxzvu 2024-03-21T12:35:20.817 INFO:teuthology.orchestra.run.smithi022.stdout: 5 14440 24.25 MB/sec warmup 20 sec latency 25.596 ms 2024-03-21T12:35:21.817 INFO:teuthology.orchestra.run.smithi022.stdout: 5 14919 23.27 MB/sec warmup 21 sec latency 29.411 ms 2024-03-21T12:35:22.403 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:35:22 smithi022 ceph-mon[24777]: pgmap v245: 97 pgs: 97 active+clean; 157 MiB data, 466 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 13 MiB/s wr, 1.23k op/s 2024-03-21T12:35:22.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:35:22 smithi189 ceph-mon[27080]: pgmap v245: 97 pgs: 97 active+clean; 157 MiB data, 466 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 13 MiB/s wr, 1.23k op/s 2024-03-21T12:35:22.817 INFO:teuthology.orchestra.run.smithi022.stdout: 5 15570 22.91 MB/sec warmup 22 sec latency 32.343 ms 2024-03-21T12:35:23.817 INFO:teuthology.orchestra.run.smithi022.stdout: 5 16518 22.62 MB/sec warmup 23 sec latency 23.705 ms 2024-03-21T12:35:24.135 INFO:teuthology.orchestra.run.smithi022.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:960fbbe4645c2becc8084669be194c95183c7753 2024-03-21T12:35:24.414 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:35:24 smithi022 ceph-mon[24777]: pgmap v246: 97 pgs: 97 active+clean; 157 MiB data, 466 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 9.1 MiB/s wr, 844 op/s 2024-03-21T12:35:24.415 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:35:24 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:35:24.415 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:35:24 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:35:24.415 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:35:24 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:35:24.415 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:35:24 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:35:24.507 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:35:24 smithi189 ceph-mon[27080]: pgmap v246: 97 pgs: 97 active+clean; 157 MiB data, 466 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 9.1 MiB/s wr, 844 op/s 2024-03-21T12:35:24.507 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:35:24 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:35:24.507 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:35:24 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:35:24.507 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:35:24 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:35:24.507 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:35:24 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:35:24.817 INFO:teuthology.orchestra.run.smithi022.stdout: 5 17305 23.03 MB/sec warmup 24 sec latency 18.713 ms 2024-03-21T12:35:25.291 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-03-21T12:35:25.292 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi022.front.sepia.ceph.com 2024-03-21T12:35:25.292 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -e sha1=960fbbe4645c2becc8084669be194c95183c7753 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done' 2024-03-21T12:35:25.817 INFO:teuthology.orchestra.run.smithi022.stdout: 5 17993 23.22 MB/sec warmup 25 sec latency 19.831 ms 2024-03-21T12:35:25.921 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:35:25 smithi022 ceph-mon[24777]: from='client.14602 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:960fbbe4645c2becc8084669be194c95183c7753", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:35:25.921 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:35:25 smithi022 ceph-mon[24777]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:960fbbe4645c2becc8084669be194c95183c7753 2024-03-21T12:35:25.921 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:35:25 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:35:25.922 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:35:25 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:35:25.922 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:35:25 smithi022 ceph-mon[24777]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:960fbbe4645c2becc8084669be194c95183c7753 2024-03-21T12:35:26.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:35:25 smithi189 ceph-mon[27080]: from='client.14602 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:960fbbe4645c2becc8084669be194c95183c7753", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:35:26.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:35:25 smithi189 ceph-mon[27080]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:960fbbe4645c2becc8084669be194c95183c7753 2024-03-21T12:35:26.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:35:25 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:35:26.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:35:25 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:35:26.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:35:25 smithi189 ceph-mon[27080]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:960fbbe4645c2becc8084669be194c95183c7753 2024-03-21T12:35:26.817 INFO:teuthology.orchestra.run.smithi022.stdout: 5 18449 22.47 MB/sec warmup 26 sec latency 19.798 ms 2024-03-21T12:35:26.927 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:35:26 smithi022 ceph-mon[24777]: pgmap v247: 97 pgs: 97 active+clean; 166 MiB data, 525 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 11 MiB/s wr, 1.09k op/s 2024-03-21T12:35:27.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:35:26 smithi189 ceph-mon[27080]: pgmap v247: 97 pgs: 97 active+clean; 166 MiB data, 525 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 11 MiB/s wr, 1.09k op/s 2024-03-21T12:35:27.817 INFO:teuthology.orchestra.run.smithi022.stdout: 5 18916 22.06 MB/sec warmup 27 sec latency 32.165 ms 2024-03-21T12:35:28.806 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:35:28 smithi022 ceph-mon[24777]: pgmap v248: 97 pgs: 97 active+clean; 168 MiB data, 560 MiB used, 715 GiB / 715 GiB avail; 10 KiB/s rd, 8.6 MiB/s wr, 776 op/s 2024-03-21T12:35:28.817 INFO:teuthology.orchestra.run.smithi022.stdout: 5 19531 21.77 MB/sec warmup 28 sec latency 35.628 ms 2024-03-21T12:35:29.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:35:28 smithi189 ceph-mon[27080]: pgmap v248: 97 pgs: 97 active+clean; 168 MiB data, 560 MiB used, 715 GiB / 715 GiB avail; 10 KiB/s rd, 8.6 MiB/s wr, 776 op/s 2024-03-21T12:35:29.045 INFO:teuthology.orchestra.run.smithi022.stdout:true 2024-03-21T12:35:29.817 INFO:teuthology.orchestra.run.smithi022.stdout: 5 20125 21.27 MB/sec warmup 29 sec latency 35.088 ms 2024-03-21T12:35:30.183 INFO:teuthology.orchestra.run.smithi022.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-21T12:35:30.183 INFO:teuthology.orchestra.run.smithi022.stdout:alertmanager.smithi022 smithi022 *:9093,9094 running (5m) 31s ago 6m 24.2M - ba2b418f427c 8fbe5dbdce93 2024-03-21T12:35:30.183 INFO:teuthology.orchestra.run.smithi022.stdout:crash.smithi022 smithi022 running (6m) 31s ago 6m 6405k - 17.2.0 e1d6a67b021e 47deb70fd250 2024-03-21T12:35:30.184 INFO:teuthology.orchestra.run.smithi022.stdout:crash.smithi189 smithi189 running (6m) 33s ago 6m 6405k - 17.2.0 e1d6a67b021e 9a2e0e9b1809 2024-03-21T12:35:30.184 INFO:teuthology.orchestra.run.smithi022.stdout:grafana.smithi022 smithi022 *:3000 running (5m) 31s ago 6m 52.7M - 8.3.5 dad864ee21e9 3c9ff131fc6b 2024-03-21T12:35:30.184 INFO:teuthology.orchestra.run.smithi022.stdout:mds.foofs.smithi022.bcnxho smithi022 running (63s) 31s ago 62s 13.2M - 17.2.0 e1d6a67b021e db26b67b8ee7 2024-03-21T12:35:30.184 INFO:teuthology.orchestra.run.smithi022.stdout:mds.foofs.smithi189.lluvko smithi189 running (60s) 33s ago 60s 14.5M - 17.2.0 e1d6a67b021e cf153aff1387 2024-03-21T12:35:30.184 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.smithi022.fektzs smithi022 *:9283 running (8m) 31s ago 8m 459M - 17.2.0 e1d6a67b021e ca06c9c2fb41 2024-03-21T12:35:30.184 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.smithi189.cnxzvu smithi189 *:8443,9283 running (6m) 33s ago 6m 413M - 17.2.0 e1d6a67b021e 5ea3c42ad934 2024-03-21T12:35:30.184 INFO:teuthology.orchestra.run.smithi022.stdout:mon.smithi022 smithi022 running (8m) 31s ago 8m 47.1M 2048M 17.2.0 e1d6a67b021e 0d248775e0e5 2024-03-21T12:35:30.184 INFO:teuthology.orchestra.run.smithi022.stdout:mon.smithi189 smithi189 running (6m) 33s ago 6m 37.2M 2048M 17.2.0 e1d6a67b021e 82da5f3cd49a 2024-03-21T12:35:30.184 INFO:teuthology.orchestra.run.smithi022.stdout:nfs.foo.0.0.smithi022.ynhbbq smithi022 *:2049 running (42s) 31s ago 41s 49.3M - 4.0 e1d6a67b021e 33e48e64c60b 2024-03-21T12:35:30.184 INFO:teuthology.orchestra.run.smithi022.stdout:nfs.foo.1.0.smithi189.yzsqml smithi189 *:2049 running (36s) 33s ago 36s 48.6M - 4.0 e1d6a67b021e 48fab2cbdde2 2024-03-21T12:35:30.184 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.smithi022 smithi022 *:9100 running (6m) 31s ago 6m 17.4M - 1dbe0e931976 cea183555968 2024-03-21T12:35:30.184 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.smithi189 smithi189 *:9100 running (5m) 33s ago 5m 19.5M - 1dbe0e931976 5db95224aff9 2024-03-21T12:35:30.184 INFO:teuthology.orchestra.run.smithi022.stdout:osd.0 smithi022 running (5m) 31s ago 5m 42.8M 3217M 17.2.0 e1d6a67b021e 54e6631bd10f 2024-03-21T12:35:30.185 INFO:teuthology.orchestra.run.smithi022.stdout:osd.1 smithi022 running (4m) 31s ago 4m 42.7M 3217M 17.2.0 e1d6a67b021e 68165c5d09cb 2024-03-21T12:35:30.185 INFO:teuthology.orchestra.run.smithi022.stdout:osd.2 smithi022 running (4m) 31s ago 4m 41.1M 3217M 17.2.0 e1d6a67b021e fbec9bef8433 2024-03-21T12:35:30.185 INFO:teuthology.orchestra.run.smithi022.stdout:osd.3 smithi022 running (4m) 31s ago 4m 40.3M 3217M 17.2.0 e1d6a67b021e b1462ffc1b9e 2024-03-21T12:35:30.185 INFO:teuthology.orchestra.run.smithi022.stdout:osd.4 smithi189 running (3m) 33s ago 3m 39.3M 3984M 17.2.0 e1d6a67b021e 1bedc4eb02ed 2024-03-21T12:35:30.185 INFO:teuthology.orchestra.run.smithi022.stdout:osd.5 smithi189 running (3m) 33s ago 3m 38.6M 3984M 17.2.0 e1d6a67b021e 25fbab9786a5 2024-03-21T12:35:30.185 INFO:teuthology.orchestra.run.smithi022.stdout:osd.6 smithi189 running (2m) 33s ago 2m 39.1M 3984M 17.2.0 e1d6a67b021e 0c192fcfd65b 2024-03-21T12:35:30.185 INFO:teuthology.orchestra.run.smithi022.stdout:osd.7 smithi189 running (2m) 33s ago 2m 43.4M 3984M 17.2.0 e1d6a67b021e 3d0723e1f863 2024-03-21T12:35:30.185 INFO:teuthology.orchestra.run.smithi022.stdout:prometheus.smithi022 smithi022 *:9095 running (5m) 31s ago 5m 63.3M - 514e6a882f6e 20a0eef27ec8 2024-03-21T12:35:30.817 INFO:teuthology.orchestra.run.smithi022.stdout: 5 20621 21.27 MB/sec warmup 30 sec latency 28.894 ms 2024-03-21T12:35:30.849 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-03-21T12:35:30.850 INFO:teuthology.orchestra.run.smithi022.stdout: "mon": { 2024-03-21T12:35:30.850 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-21T12:35:30.850 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:35:30.850 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr": { 2024-03-21T12:35:30.850 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-21T12:35:30.850 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:35:30.850 INFO:teuthology.orchestra.run.smithi022.stdout: "osd": { 2024-03-21T12:35:30.850 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-03-21T12:35:30.850 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:35:30.850 INFO:teuthology.orchestra.run.smithi022.stdout: "mds": { 2024-03-21T12:35:30.850 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-21T12:35:30.850 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:35:30.850 INFO:teuthology.orchestra.run.smithi022.stdout: "overall": { 2024-03-21T12:35:30.850 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-03-21T12:35:30.851 INFO:teuthology.orchestra.run.smithi022.stdout: } 2024-03-21T12:35:30.851 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-03-21T12:35:30.963 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:35:30 smithi022 ceph-mon[24777]: pgmap v249: 97 pgs: 97 active+clean; 173 MiB data, 576 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 8.2 MiB/s wr, 780 op/s 2024-03-21T12:35:30.963 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:35:30 smithi022 ceph-mon[24777]: from='client.24393 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:35:30.963 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:35:30 smithi022 ceph-mon[24777]: from='client.24397 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:35:31.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:35:30 smithi189 ceph-mon[27080]: pgmap v249: 97 pgs: 97 active+clean; 173 MiB data, 576 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 8.2 MiB/s wr, 780 op/s 2024-03-21T12:35:31.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:35:30 smithi189 ceph-mon[27080]: from='client.24393 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:35:31.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:35:30 smithi189 ceph-mon[27080]: from='client.24397 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:35:31.440 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-03-21T12:35:31.440 INFO:teuthology.orchestra.run.smithi022.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:960fbbe4645c2becc8084669be194c95183c7753", 2024-03-21T12:35:31.440 INFO:teuthology.orchestra.run.smithi022.stdout: "in_progress": true, 2024-03-21T12:35:31.440 INFO:teuthology.orchestra.run.smithi022.stdout: "services_complete": [], 2024-03-21T12:35:31.440 INFO:teuthology.orchestra.run.smithi022.stdout: "progress": "", 2024-03-21T12:35:31.440 INFO:teuthology.orchestra.run.smithi022.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:960fbbe4645c2becc8084669be194c95183c7753 image" 2024-03-21T12:35:31.441 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-03-21T12:35:31.799 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:35:31 smithi022 ceph-mon[24777]: from='client.14612 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:35:31.800 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:35:31 smithi022 ceph-mon[24777]: from='client.? 172.21.15.22:0/1965976141' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:35:31.817 INFO:teuthology.orchestra.run.smithi022.stdout: 5 21321 21.48 MB/sec warmup 31 sec latency 20.767 ms 2024-03-21T12:35:32.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:35:31 smithi189 ceph-mon[27080]: from='client.14612 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:35:32.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:35:31 smithi189 ceph-mon[27080]: from='client.? 172.21.15.22:0/1965976141' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:35:32.099 INFO:teuthology.orchestra.run.smithi022.stdout:HEALTH_OK 2024-03-21T12:35:32.806 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:35:32 smithi022 ceph-mon[24777]: pgmap v250: 97 pgs: 97 active+clean; 176 MiB data, 627 MiB used, 715 GiB / 715 GiB avail; 10 KiB/s rd, 10 MiB/s wr, 979 op/s 2024-03-21T12:35:32.806 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:35:32 smithi022 ceph-mon[24777]: from='client.14618 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:35:32.806 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:35:32 smithi022 ceph-mon[24777]: from='client.? 172.21.15.22:0/2561582170' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-21T12:35:32.817 INFO:teuthology.orchestra.run.smithi022.stdout: 5 21855 21.22 MB/sec warmup 32 sec latency 17.226 ms 2024-03-21T12:35:33.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:35:32 smithi189 ceph-mon[27080]: pgmap v250: 97 pgs: 97 active+clean; 176 MiB data, 627 MiB used, 715 GiB / 715 GiB avail; 10 KiB/s rd, 10 MiB/s wr, 979 op/s 2024-03-21T12:35:33.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:35:32 smithi189 ceph-mon[27080]: from='client.14618 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:35:33.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:35:32 smithi189 ceph-mon[27080]: from='client.? 172.21.15.22:0/2561582170' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-21T12:35:33.817 INFO:teuthology.orchestra.run.smithi022.stdout: 5 22476 20.89 MB/sec warmup 33 sec latency 20.149 ms 2024-03-21T12:35:34.817 INFO:teuthology.orchestra.run.smithi022.stdout: 5 23243 20.80 MB/sec warmup 34 sec latency 19.824 ms 2024-03-21T12:35:34.820 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:35:34 smithi022 ceph-mon[24777]: pgmap v251: 97 pgs: 97 active+clean; 176 MiB data, 627 MiB used, 715 GiB / 715 GiB avail; 10 KiB/s rd, 7.2 MiB/s wr, 705 op/s 2024-03-21T12:35:35.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:35:34 smithi189 ceph-mon[27080]: pgmap v251: 97 pgs: 97 active+clean; 176 MiB data, 627 MiB used, 715 GiB / 715 GiB avail; 10 KiB/s rd, 7.2 MiB/s wr, 705 op/s 2024-03-21T12:35:35.817 INFO:teuthology.orchestra.run.smithi022.stdout: 5 23799 20.51 MB/sec warmup 35 sec latency 46.672 ms 2024-03-21T12:35:36.818 INFO:teuthology.orchestra.run.smithi022.stdout: 5 24419 20.72 MB/sec warmup 36 sec latency 19.827 ms 2024-03-21T12:35:36.846 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:35:36 smithi022 ceph-mon[24777]: pgmap v252: 97 pgs: 97 active+clean; 189 MiB data, 650 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 9.9 MiB/s wr, 956 op/s 2024-03-21T12:35:37.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:35:36 smithi189 ceph-mon[27080]: pgmap v252: 97 pgs: 97 active+clean; 189 MiB data, 650 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 9.9 MiB/s wr, 956 op/s 2024-03-21T12:35:37.818 INFO:teuthology.orchestra.run.smithi022.stdout: 5 24942 20.67 MB/sec warmup 37 sec latency 40.526 ms 2024-03-21T12:35:38.818 INFO:teuthology.orchestra.run.smithi022.stdout: 5 25276 20.36 MB/sec warmup 38 sec latency 31.011 ms 2024-03-21T12:35:38.876 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:35:38 smithi022 ceph-mon[24777]: pgmap v253: 97 pgs: 97 active+clean; 188 MiB data, 677 MiB used, 715 GiB / 715 GiB avail; 9.6 KiB/s rd, 8.5 MiB/s wr, 760 op/s 2024-03-21T12:35:39.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:35:38 smithi189 ceph-mon[27080]: pgmap v253: 97 pgs: 97 active+clean; 188 MiB data, 677 MiB used, 715 GiB / 715 GiB avail; 9.6 KiB/s rd, 8.5 MiB/s wr, 760 op/s 2024-03-21T12:35:39.817 INFO:teuthology.orchestra.run.smithi022.stdout: 5 25693 19.94 MB/sec warmup 39 sec latency 32.135 ms 2024-03-21T12:35:40.818 INFO:teuthology.orchestra.run.smithi022.stdout: 5 26218 19.77 MB/sec warmup 40 sec latency 29.300 ms 2024-03-21T12:35:40.890 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:35:40 smithi022 ceph-mon[24777]: pgmap v254: 97 pgs: 97 active+clean; 190 MiB data, 726 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 7.8 MiB/s wr, 694 op/s 2024-03-21T12:35:41.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:35:40 smithi189 ceph-mon[27080]: pgmap v254: 97 pgs: 97 active+clean; 190 MiB data, 726 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 7.8 MiB/s wr, 694 op/s 2024-03-21T12:35:41.818 INFO:teuthology.orchestra.run.smithi022.stdout: 5 26910 19.59 MB/sec warmup 41 sec latency 26.508 ms 2024-03-21T12:35:42.818 INFO:teuthology.orchestra.run.smithi022.stdout: 5 27397 19.40 MB/sec warmup 42 sec latency 50.119 ms 2024-03-21T12:35:42.912 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:35:42 smithi022 ceph-mon[24777]: pgmap v255: 97 pgs: 97 active+clean; 204 MiB data, 810 MiB used, 714 GiB / 715 GiB avail; 9.2 KiB/s rd, 9.9 MiB/s wr, 918 op/s 2024-03-21T12:35:43.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:35:42 smithi189 ceph-mon[27080]: pgmap v255: 97 pgs: 97 active+clean; 204 MiB data, 810 MiB used, 714 GiB / 715 GiB avail; 9.2 KiB/s rd, 9.9 MiB/s wr, 918 op/s 2024-03-21T12:35:43.818 INFO:teuthology.orchestra.run.smithi022.stdout: 5 28286 19.78 MB/sec warmup 43 sec latency 24.003 ms 2024-03-21T12:35:44.818 INFO:teuthology.orchestra.run.smithi022.stdout: 5 28975 19.78 MB/sec warmup 44 sec latency 12.239 ms 2024-03-21T12:35:44.922 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:35:44 smithi022 ceph-mon[24777]: pgmap v256: 97 pgs: 97 active+clean; 204 MiB data, 810 MiB used, 714 GiB / 715 GiB avail; 9.2 KiB/s rd, 7.5 MiB/s wr, 692 op/s 2024-03-21T12:35:45.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:35:44 smithi189 ceph-mon[27080]: pgmap v256: 97 pgs: 97 active+clean; 204 MiB data, 810 MiB used, 714 GiB / 715 GiB avail; 9.2 KiB/s rd, 7.5 MiB/s wr, 692 op/s 2024-03-21T12:35:45.818 INFO:teuthology.orchestra.run.smithi022.stdout: 5 29666 19.63 MB/sec warmup 45 sec latency 15.271 ms 2024-03-21T12:35:46.818 INFO:teuthology.orchestra.run.smithi022.stdout: 5 30253 19.52 MB/sec warmup 46 sec latency 33.252 ms 2024-03-21T12:35:46.944 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:35:46 smithi022 ceph-mon[24777]: pgmap v257: 97 pgs: 97 active+clean; 207 MiB data, 860 MiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 9.5 MiB/s wr, 872 op/s 2024-03-21T12:35:47.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:35:46 smithi189 ceph-mon[27080]: pgmap v257: 97 pgs: 97 active+clean; 207 MiB data, 860 MiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 9.5 MiB/s wr, 872 op/s 2024-03-21T12:35:47.818 INFO:teuthology.orchestra.run.smithi022.stdout: 5 30766 19.22 MB/sec warmup 47 sec latency 44.587 ms 2024-03-21T12:35:48.818 INFO:teuthology.orchestra.run.smithi022.stdout: 5 31099 19.06 MB/sec warmup 48 sec latency 33.864 ms 2024-03-21T12:35:49.026 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:35:48 smithi022 ceph-mon[24777]: pgmap v258: 97 pgs: 97 active+clean; 211 MiB data, 880 MiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 8.3 MiB/s wr, 756 op/s 2024-03-21T12:35:49.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:35:48 smithi189 ceph-mon[27080]: pgmap v258: 97 pgs: 97 active+clean; 211 MiB data, 880 MiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 8.3 MiB/s wr, 756 op/s 2024-03-21T12:35:49.818 INFO:teuthology.orchestra.run.smithi022.stdout: 5 31735 19.23 MB/sec warmup 49 sec latency 28.501 ms 2024-03-21T12:35:50.818 INFO:teuthology.orchestra.run.smithi022.stdout: 5 32211 19.26 MB/sec warmup 50 sec latency 31.984 ms 2024-03-21T12:35:51.043 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:35:50 smithi022 ceph-mon[24777]: pgmap v259: 97 pgs: 97 active+clean; 210 MiB data, 886 MiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 7.8 MiB/s wr, 731 op/s 2024-03-21T12:35:51.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:35:50 smithi189 ceph-mon[27080]: pgmap v259: 97 pgs: 97 active+clean; 210 MiB data, 886 MiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 7.8 MiB/s wr, 731 op/s 2024-03-21T12:35:51.818 INFO:teuthology.orchestra.run.smithi022.stdout: 5 32691 18.98 MB/sec warmup 51 sec latency 30.083 ms 2024-03-21T12:35:52.818 INFO:teuthology.orchestra.run.smithi022.stdout: 5 33343 18.89 MB/sec warmup 52 sec latency 28.806 ms 2024-03-21T12:35:52.911 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:35:52 smithi022 ceph-mon[24777]: pgmap v260: 97 pgs: 97 active+clean; 214 MiB data, 932 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.7 MiB/s wr, 947 op/s 2024-03-21T12:35:53.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:35:52 smithi189 ceph-mon[27080]: pgmap v260: 97 pgs: 97 active+clean; 214 MiB data, 932 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.7 MiB/s wr, 947 op/s 2024-03-21T12:35:53.818 INFO:teuthology.orchestra.run.smithi022.stdout: 5 33906 18.75 MB/sec warmup 53 sec latency 29.662 ms 2024-03-21T12:35:54.818 INFO:teuthology.orchestra.run.smithi022.stdout: 5 34425 18.60 MB/sec warmup 54 sec latency 74.595 ms 2024-03-21T12:35:54.942 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:35:54 smithi022 ceph-mon[24777]: pgmap v261: 97 pgs: 97 active+clean; 214 MiB data, 932 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 6.6 MiB/s wr, 612 op/s 2024-03-21T12:35:55.087 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:35:54 smithi189 ceph-mon[27080]: pgmap v261: 97 pgs: 97 active+clean; 214 MiB data, 932 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 6.6 MiB/s wr, 612 op/s 2024-03-21T12:35:55.818 INFO:teuthology.orchestra.run.smithi022.stdout: 5 34971 18.69 MB/sec warmup 55 sec latency 29.537 ms 2024-03-21T12:35:56.818 INFO:teuthology.orchestra.run.smithi022.stdout: 5 35599 18.78 MB/sec warmup 56 sec latency 43.137 ms 2024-03-21T12:35:56.956 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:35:56 smithi022 ceph-mon[24777]: pgmap v262: 97 pgs: 97 active+clean; 225 MiB data, 978 MiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 9.1 MiB/s wr, 889 op/s 2024-03-21T12:35:57.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:35:56 smithi189 ceph-mon[27080]: pgmap v262: 97 pgs: 97 active+clean; 225 MiB data, 978 MiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 9.1 MiB/s wr, 889 op/s 2024-03-21T12:35:57.818 INFO:teuthology.orchestra.run.smithi022.stdout: 5 36055 18.63 MB/sec warmup 57 sec latency 32.289 ms 2024-03-21T12:35:58.820 INFO:teuthology.orchestra.run.smithi022.stdout: 5 36447 18.37 MB/sec warmup 58 sec latency 58.670 ms 2024-03-21T12:35:58.990 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:35:58 smithi022 ceph-mon[24777]: pgmap v263: 97 pgs: 97 active+clean; 226 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 9.2 KiB/s rd, 7.8 MiB/s wr, 753 op/s 2024-03-21T12:35:59.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:35:58 smithi189 ceph-mon[27080]: pgmap v263: 97 pgs: 97 active+clean; 226 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 9.2 KiB/s rd, 7.8 MiB/s wr, 753 op/s 2024-03-21T12:35:59.820 INFO:teuthology.orchestra.run.smithi022.stdout: 5 37146 18.43 MB/sec warmup 59 sec latency 31.716 ms 2024-03-21T12:36:00.820 INFO:teuthology.orchestra.run.smithi022.stdout: 5 37942 18.32 MB/sec warmup 60 sec latency 36.670 ms 2024-03-21T12:36:01.002 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:00 smithi022 ceph-mon[24777]: pgmap v264: 97 pgs: 97 active+clean; 225 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 6.9 MiB/s wr, 641 op/s 2024-03-21T12:36:01.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:00 smithi189 ceph-mon[27080]: pgmap v264: 97 pgs: 97 active+clean; 225 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 6.9 MiB/s wr, 641 op/s 2024-03-21T12:36:01.820 INFO:teuthology.orchestra.run.smithi022.stdout: 5 38584 18.49 MB/sec warmup 61 sec latency 29.245 ms 2024-03-21T12:36:02.633 INFO:teuthology.orchestra.run.smithi022.stdout:true 2024-03-21T12:36:02.820 INFO:teuthology.orchestra.run.smithi022.stdout: 5 39196 18.56 MB/sec warmup 62 sec latency 33.188 ms 2024-03-21T12:36:03.031 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:02 smithi022 ceph-mon[24777]: pgmap v265: 97 pgs: 97 active+clean; 237 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 7.6 KiB/s rd, 9.3 MiB/s wr, 909 op/s 2024-03-21T12:36:03.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:02 smithi189 ceph-mon[27080]: pgmap v265: 97 pgs: 97 active+clean; 237 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 7.6 KiB/s rd, 9.3 MiB/s wr, 909 op/s 2024-03-21T12:36:03.660 INFO:teuthology.orchestra.run.smithi022.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-21T12:36:03.660 INFO:teuthology.orchestra.run.smithi022.stdout:alertmanager.smithi022 smithi022 *:9093,9094 running (5m) 64s ago 7m 24.2M - ba2b418f427c 8fbe5dbdce93 2024-03-21T12:36:03.660 INFO:teuthology.orchestra.run.smithi022.stdout:crash.smithi022 smithi022 running (7m) 64s ago 7m 6405k - 17.2.0 e1d6a67b021e 47deb70fd250 2024-03-21T12:36:03.660 INFO:teuthology.orchestra.run.smithi022.stdout:crash.smithi189 smithi189 running (6m) 66s ago 6m 6405k - 17.2.0 e1d6a67b021e 9a2e0e9b1809 2024-03-21T12:36:03.660 INFO:teuthology.orchestra.run.smithi022.stdout:grafana.smithi022 smithi022 *:3000 running (5m) 64s ago 7m 52.7M - 8.3.5 dad864ee21e9 3c9ff131fc6b 2024-03-21T12:36:03.660 INFO:teuthology.orchestra.run.smithi022.stdout:mds.foofs.smithi022.bcnxho smithi022 running (97s) 64s ago 96s 13.2M - 17.2.0 e1d6a67b021e db26b67b8ee7 2024-03-21T12:36:03.660 INFO:teuthology.orchestra.run.smithi022.stdout:mds.foofs.smithi189.lluvko smithi189 running (94s) 66s ago 93s 14.5M - 17.2.0 e1d6a67b021e cf153aff1387 2024-03-21T12:36:03.660 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.smithi022.fektzs smithi022 *:9283 running (8m) 64s ago 8m 459M - 17.2.0 e1d6a67b021e ca06c9c2fb41 2024-03-21T12:36:03.660 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.smithi189.cnxzvu smithi189 *:8443,9283 running (6m) 66s ago 6m 413M - 17.2.0 e1d6a67b021e 5ea3c42ad934 2024-03-21T12:36:03.660 INFO:teuthology.orchestra.run.smithi022.stdout:mon.smithi022 smithi022 running (8m) 64s ago 8m 47.1M 2048M 17.2.0 e1d6a67b021e 0d248775e0e5 2024-03-21T12:36:03.660 INFO:teuthology.orchestra.run.smithi022.stdout:mon.smithi189 smithi189 running (6m) 66s ago 6m 37.2M 2048M 17.2.0 e1d6a67b021e 82da5f3cd49a 2024-03-21T12:36:03.660 INFO:teuthology.orchestra.run.smithi022.stdout:nfs.foo.0.0.smithi022.ynhbbq smithi022 *:2049 running (75s) 64s ago 75s 49.3M - 4.0 e1d6a67b021e 33e48e64c60b 2024-03-21T12:36:03.661 INFO:teuthology.orchestra.run.smithi022.stdout:nfs.foo.1.0.smithi189.yzsqml smithi189 *:2049 running (70s) 66s ago 69s 48.6M - 4.0 e1d6a67b021e 48fab2cbdde2 2024-03-21T12:36:03.661 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.smithi022 smithi022 *:9100 running (7m) 64s ago 7m 17.4M - 1dbe0e931976 cea183555968 2024-03-21T12:36:03.661 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.smithi189 smithi189 *:9100 running (6m) 66s ago 6m 19.5M - 1dbe0e931976 5db95224aff9 2024-03-21T12:36:03.661 INFO:teuthology.orchestra.run.smithi022.stdout:osd.0 smithi022 running (6m) 64s ago 5m 42.8M 3217M 17.2.0 e1d6a67b021e 54e6631bd10f 2024-03-21T12:36:03.661 INFO:teuthology.orchestra.run.smithi022.stdout:osd.1 smithi022 running (5m) 64s ago 5m 42.7M 3217M 17.2.0 e1d6a67b021e 68165c5d09cb 2024-03-21T12:36:03.661 INFO:teuthology.orchestra.run.smithi022.stdout:osd.2 smithi022 running (5m) 64s ago 5m 41.1M 3217M 17.2.0 e1d6a67b021e fbec9bef8433 2024-03-21T12:36:03.661 INFO:teuthology.orchestra.run.smithi022.stdout:osd.3 smithi022 running (4m) 64s ago 4m 40.3M 3217M 17.2.0 e1d6a67b021e b1462ffc1b9e 2024-03-21T12:36:03.661 INFO:teuthology.orchestra.run.smithi022.stdout:osd.4 smithi189 running (4m) 66s ago 4m 39.3M 3984M 17.2.0 e1d6a67b021e 1bedc4eb02ed 2024-03-21T12:36:03.661 INFO:teuthology.orchestra.run.smithi022.stdout:osd.5 smithi189 running (3m) 66s ago 3m 38.6M 3984M 17.2.0 e1d6a67b021e 25fbab9786a5 2024-03-21T12:36:03.661 INFO:teuthology.orchestra.run.smithi022.stdout:osd.6 smithi189 running (3m) 66s ago 3m 39.1M 3984M 17.2.0 e1d6a67b021e 0c192fcfd65b 2024-03-21T12:36:03.661 INFO:teuthology.orchestra.run.smithi022.stdout:osd.7 smithi189 running (3m) 66s ago 3m 43.4M 3984M 17.2.0 e1d6a67b021e 3d0723e1f863 2024-03-21T12:36:03.661 INFO:teuthology.orchestra.run.smithi022.stdout:prometheus.smithi022 smithi022 *:9095 running (6m) 64s ago 6m 63.3M - 514e6a882f6e 20a0eef27ec8 2024-03-21T12:36:03.820 INFO:teuthology.orchestra.run.smithi022.stdout: 5 39546 18.37 MB/sec warmup 63 sec latency 33.873 ms 2024-03-21T12:36:04.037 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:03 smithi022 ceph-mon[24777]: from='client.14626 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:36:04.037 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:03 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:36:04.038 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:03 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:36:04.038 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:03 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:36:04.287 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:03 smithi189 ceph-mon[27080]: from='client.14626 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:36:04.287 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:03 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:36:04.287 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:03 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:36:04.288 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:03 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:36:04.322 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-03-21T12:36:04.322 INFO:teuthology.orchestra.run.smithi022.stdout: "mon": { 2024-03-21T12:36:04.322 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-21T12:36:04.322 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:36:04.323 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr": { 2024-03-21T12:36:04.323 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-21T12:36:04.323 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:36:04.323 INFO:teuthology.orchestra.run.smithi022.stdout: "osd": { 2024-03-21T12:36:04.323 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-03-21T12:36:04.323 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:36:04.323 INFO:teuthology.orchestra.run.smithi022.stdout: "mds": { 2024-03-21T12:36:04.323 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-21T12:36:04.323 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:36:04.323 INFO:teuthology.orchestra.run.smithi022.stdout: "overall": { 2024-03-21T12:36:04.323 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-03-21T12:36:04.323 INFO:teuthology.orchestra.run.smithi022.stdout: } 2024-03-21T12:36:04.323 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-03-21T12:36:04.820 INFO:teuthology.orchestra.run.smithi022.stdout: 5 39999 18.19 MB/sec warmup 64 sec latency 20.762 ms 2024-03-21T12:36:04.869 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-03-21T12:36:04.870 INFO:teuthology.orchestra.run.smithi022.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93", 2024-03-21T12:36:04.870 INFO:teuthology.orchestra.run.smithi022.stdout: "in_progress": true, 2024-03-21T12:36:04.870 INFO:teuthology.orchestra.run.smithi022.stdout: "services_complete": [], 2024-03-21T12:36:04.870 INFO:teuthology.orchestra.run.smithi022.stdout: "progress": "0/23 daemons upgraded", 2024-03-21T12:36:04.870 INFO:teuthology.orchestra.run.smithi022.stdout: "message": "Pulling quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93 image on host smithi189" 2024-03-21T12:36:04.870 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-03-21T12:36:05.041 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:04 smithi022 ceph-mon[24777]: pgmap v266: 97 pgs: 97 active+clean; 237 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 7.6 KiB/s rd, 6.7 MiB/s wr, 637 op/s 2024-03-21T12:36:05.042 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:04 smithi022 ceph-mon[24777]: Upgrade: Target is version 19.0.0-2267-g960fbbe4 (unknown) 2024-03-21T12:36:05.042 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:04 smithi022 ceph-mon[24777]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, digests ['quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93'] 2024-03-21T12:36:05.042 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:04 smithi022 ceph-mon[24777]: Upgrade: Need to upgrade myself (mgr.smithi022.fektzs) 2024-03-21T12:36:05.042 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:04 smithi022 ceph-mon[24777]: from='client.14630 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:36:05.042 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:04 smithi022 ceph-mon[24777]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93 on smithi189 2024-03-21T12:36:05.042 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:04 smithi022 ceph-mon[24777]: from='client.14634 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:36:05.042 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:04 smithi022 ceph-mon[24777]: from='client.? 172.21.15.22:0/838378790' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:36:05.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:04 smithi189 ceph-mon[27080]: pgmap v266: 97 pgs: 97 active+clean; 237 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 7.6 KiB/s rd, 6.7 MiB/s wr, 637 op/s 2024-03-21T12:36:05.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:04 smithi189 ceph-mon[27080]: Upgrade: Target is version 19.0.0-2267-g960fbbe4 (unknown) 2024-03-21T12:36:05.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:04 smithi189 ceph-mon[27080]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, digests ['quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93'] 2024-03-21T12:36:05.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:04 smithi189 ceph-mon[27080]: Upgrade: Need to upgrade myself (mgr.smithi022.fektzs) 2024-03-21T12:36:05.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:04 smithi189 ceph-mon[27080]: from='client.14630 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:36:05.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:04 smithi189 ceph-mon[27080]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93 on smithi189 2024-03-21T12:36:05.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:04 smithi189 ceph-mon[27080]: from='client.14634 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:36:05.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:04 smithi189 ceph-mon[27080]: from='client.? 172.21.15.22:0/838378790' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:36:05.544 INFO:teuthology.orchestra.run.smithi022.stdout:HEALTH_OK 2024-03-21T12:36:05.820 INFO:teuthology.orchestra.run.smithi022.stdout: 5 40620 18.15 MB/sec warmup 65 sec latency 26.183 ms 2024-03-21T12:36:06.048 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:05 smithi022 ceph-mon[24777]: from='client.24427 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:36:06.048 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:05 smithi022 ceph-mon[24777]: from='client.? 172.21.15.22:0/3052354635' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-21T12:36:06.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:05 smithi189 ceph-mon[27080]: from='client.24427 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:36:06.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:05 smithi189 ceph-mon[27080]: from='client.? 172.21.15.22:0/3052354635' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-21T12:36:06.820 INFO:teuthology.orchestra.run.smithi022.stdout: 5 41448 18.08 MB/sec warmup 66 sec latency 31.942 ms 2024-03-21T12:36:07.056 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:06 smithi022 ceph-mon[24777]: pgmap v267: 97 pgs: 97 active+clean; 245 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 9.4 MiB/s wr, 913 op/s 2024-03-21T12:36:07.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:06 smithi189 ceph-mon[27080]: pgmap v267: 97 pgs: 97 active+clean; 245 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 9.4 MiB/s wr, 913 op/s 2024-03-21T12:36:07.820 INFO:teuthology.orchestra.run.smithi022.stdout: 5 42004 18.15 MB/sec warmup 67 sec latency 21.131 ms 2024-03-21T12:36:08.066 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:07 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi022.fektzs/trash_purge_schedule"}]: dispatch 2024-03-21T12:36:08.066 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:07 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi022.fektzs/mirror_snapshot_schedule"}]: dispatch 2024-03-21T12:36:08.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:07 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi022.fektzs/trash_purge_schedule"}]: dispatch 2024-03-21T12:36:08.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:07 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi022.fektzs/mirror_snapshot_schedule"}]: dispatch 2024-03-21T12:36:08.820 INFO:teuthology.orchestra.run.smithi022.stdout: 5 42494 18.13 MB/sec warmup 68 sec latency 51.593 ms 2024-03-21T12:36:08.869 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:08 smithi022 ceph-mon[24777]: pgmap v268: 97 pgs: 97 active+clean; 245 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 9.2 KiB/s rd, 7.8 MiB/s wr, 676 op/s 2024-03-21T12:36:09.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:08 smithi189 ceph-mon[27080]: pgmap v268: 97 pgs: 97 active+clean; 245 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 9.2 KiB/s rd, 7.8 MiB/s wr, 676 op/s 2024-03-21T12:36:09.820 INFO:teuthology.orchestra.run.smithi022.stdout: 5 42964 18.14 MB/sec warmup 69 sec latency 38.156 ms 2024-03-21T12:36:10.820 INFO:teuthology.orchestra.run.smithi022.stdout: 5 43547 17.99 MB/sec warmup 70 sec latency 24.900 ms 2024-03-21T12:36:10.964 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:10 smithi022 ceph-mon[24777]: pgmap v269: 97 pgs: 97 active+clean; 247 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 7.8 MiB/s wr, 723 op/s 2024-03-21T12:36:11.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:10 smithi189 ceph-mon[27080]: pgmap v269: 97 pgs: 97 active+clean; 247 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 7.8 MiB/s wr, 723 op/s 2024-03-21T12:36:11.820 INFO:teuthology.orchestra.run.smithi022.stdout: 5 44387 18.04 MB/sec warmup 71 sec latency 17.600 ms 2024-03-21T12:36:12.820 INFO:teuthology.orchestra.run.smithi022.stdout: 5 45224 18.00 MB/sec warmup 72 sec latency 22.353 ms 2024-03-21T12:36:12.995 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:12 smithi022 ceph-mon[24777]: pgmap v270: 97 pgs: 97 active+clean; 260 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 10 MiB/s wr, 985 op/s 2024-03-21T12:36:13.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:12 smithi189 ceph-mon[27080]: pgmap v270: 97 pgs: 97 active+clean; 260 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 10 MiB/s wr, 985 op/s 2024-03-21T12:36:13.820 INFO:teuthology.orchestra.run.smithi022.stdout: 5 45876 18.14 MB/sec warmup 73 sec latency 29.322 ms 2024-03-21T12:36:14.820 INFO:teuthology.orchestra.run.smithi022.stdout: 5 46636 18.22 MB/sec warmup 74 sec latency 52.407 ms 2024-03-21T12:36:15.011 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:14 smithi022 ceph-mon[24777]: pgmap v271: 97 pgs: 97 active+clean; 260 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 7.3 MiB/s wr, 692 op/s 2024-03-21T12:36:15.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:14 smithi189 ceph-mon[27080]: pgmap v271: 97 pgs: 97 active+clean; 260 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 7.3 MiB/s wr, 692 op/s 2024-03-21T12:36:15.820 INFO:teuthology.orchestra.run.smithi022.stdout: 5 47362 18.12 MB/sec warmup 75 sec latency 37.034 ms 2024-03-21T12:36:16.820 INFO:teuthology.orchestra.run.smithi022.stdout: 5 48341 18.19 MB/sec warmup 76 sec latency 18.211 ms 2024-03-21T12:36:17.020 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:16 smithi022 ceph-mon[24777]: pgmap v272: 97 pgs: 97 active+clean; 265 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 9.8 MiB/s wr, 932 op/s 2024-03-21T12:36:17.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:16 smithi189 ceph-mon[27080]: pgmap v272: 97 pgs: 97 active+clean; 265 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 9.8 MiB/s wr, 932 op/s 2024-03-21T12:36:17.820 INFO:teuthology.orchestra.run.smithi022.stdout: 5 49154 18.32 MB/sec warmup 77 sec latency 42.214 ms 2024-03-21T12:36:18.820 INFO:teuthology.orchestra.run.smithi022.stdout: 5 50015 18.47 MB/sec warmup 78 sec latency 21.182 ms 2024-03-21T12:36:19.030 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:18 smithi022 ceph-mon[24777]: pgmap v273: 97 pgs: 97 active+clean; 271 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 8.9 MiB/s wr, 815 op/s 2024-03-21T12:36:19.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:18 smithi189 ceph-mon[27080]: pgmap v273: 97 pgs: 97 active+clean; 271 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 8.9 MiB/s wr, 815 op/s 2024-03-21T12:36:19.820 INFO:teuthology.orchestra.run.smithi022.stdout: 5 50630 18.36 MB/sec warmup 79 sec latency 32.253 ms 2024-03-21T12:36:20.820 INFO:teuthology.orchestra.run.smithi022.stdout: 5 51444 18.39 MB/sec warmup 80 sec latency 24.835 ms 2024-03-21T12:36:21.042 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:20 smithi022 ceph-mon[24777]: pgmap v274: 97 pgs: 97 active+clean; 273 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 8.9 MiB/s wr, 812 op/s 2024-03-21T12:36:21.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:20 smithi189 ceph-mon[27080]: pgmap v274: 97 pgs: 97 active+clean; 273 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 8.9 MiB/s wr, 812 op/s 2024-03-21T12:36:21.820 INFO:teuthology.orchestra.run.smithi022.stdout: 5 52318 18.36 MB/sec warmup 81 sec latency 23.268 ms 2024-03-21T12:36:22.820 INFO:teuthology.orchestra.run.smithi022.stdout: 5 53076 18.51 MB/sec warmup 82 sec latency 27.145 ms 2024-03-21T12:36:23.055 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:22 smithi022 ceph-mon[24777]: pgmap v275: 97 pgs: 97 active+clean; 285 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 12 MiB/s wr, 1.12k op/s 2024-03-21T12:36:23.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:22 smithi189 ceph-mon[27080]: pgmap v275: 97 pgs: 97 active+clean; 285 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 12 MiB/s wr, 1.12k op/s 2024-03-21T12:36:23.820 INFO:teuthology.orchestra.run.smithi022.stdout: 5 53710 18.55 MB/sec warmup 83 sec latency 61.442 ms 2024-03-21T12:36:24.820 INFO:teuthology.orchestra.run.smithi022.stdout: 5 54284 18.43 MB/sec warmup 84 sec latency 35.493 ms 2024-03-21T12:36:25.068 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:24 smithi022 ceph-mon[24777]: pgmap v276: 97 pgs: 97 active+clean; 285 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.1 MiB/s wr, 834 op/s 2024-03-21T12:36:25.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:24 smithi189 ceph-mon[27080]: pgmap v276: 97 pgs: 97 active+clean; 285 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.1 MiB/s wr, 834 op/s 2024-03-21T12:36:25.820 INFO:teuthology.orchestra.run.smithi022.stdout: 5 54883 18.36 MB/sec warmup 85 sec latency 23.154 ms 2024-03-21T12:36:25.932 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:25 smithi022 ceph-mon[24777]: pgmap v277: 97 pgs: 97 active+clean; 295 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 12 MiB/s wr, 1.19k op/s 2024-03-21T12:36:26.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:25 smithi189 ceph-mon[27080]: pgmap v277: 97 pgs: 97 active+clean; 295 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 12 MiB/s wr, 1.19k op/s 2024-03-21T12:36:26.820 INFO:teuthology.orchestra.run.smithi022.stdout: 5 55611 18.32 MB/sec warmup 86 sec latency 46.417 ms 2024-03-21T12:36:27.821 INFO:teuthology.orchestra.run.smithi022.stdout: 5 56119 18.31 MB/sec warmup 87 sec latency 28.945 ms 2024-03-21T12:36:28.405 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:28 smithi022 ceph-mon[24777]: pgmap v278: 97 pgs: 97 active+clean; 297 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 11 MiB/s wr, 998 op/s 2024-03-21T12:36:28.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:28 smithi189 ceph-mon[27080]: pgmap v278: 97 pgs: 97 active+clean; 297 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 11 MiB/s wr, 998 op/s 2024-03-21T12:36:28.820 INFO:teuthology.orchestra.run.smithi022.stdout: 5 56693 18.38 MB/sec warmup 88 sec latency 36.476 ms 2024-03-21T12:36:29.821 INFO:teuthology.orchestra.run.smithi022.stdout: 5 57160 18.36 MB/sec warmup 89 sec latency 32.552 ms 2024-03-21T12:36:30.409 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:30 smithi022 ceph-mon[24777]: pgmap v279: 97 pgs: 97 active+clean; 300 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 10 MiB/s wr, 955 op/s 2024-03-21T12:36:30.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:30 smithi189 ceph-mon[27080]: pgmap v279: 97 pgs: 97 active+clean; 300 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 10 MiB/s wr, 955 op/s 2024-03-21T12:36:30.821 INFO:teuthology.orchestra.run.smithi022.stdout: 5 57667 18.23 MB/sec warmup 90 sec latency 25.615 ms 2024-03-21T12:36:31.821 INFO:teuthology.orchestra.run.smithi022.stdout: 5 58116 18.13 MB/sec warmup 91 sec latency 39.547 ms 2024-03-21T12:36:32.460 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:32 smithi022 ceph-mon[24777]: pgmap v280: 97 pgs: 97 active+clean; 305 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 12 MiB/s wr, 1.14k op/s 2024-03-21T12:36:32.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:32 smithi189 ceph-mon[27080]: pgmap v280: 97 pgs: 97 active+clean; 305 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 12 MiB/s wr, 1.14k op/s 2024-03-21T12:36:32.821 INFO:teuthology.orchestra.run.smithi022.stdout: 5 58636 18.08 MB/sec warmup 92 sec latency 32.539 ms 2024-03-21T12:36:33.821 INFO:teuthology.orchestra.run.smithi022.stdout: 5 59373 18.07 MB/sec warmup 93 sec latency 37.058 ms 2024-03-21T12:36:34.226 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:34 smithi022 ceph-mon[24777]: pgmap v281: 97 pgs: 97 active+clean; 305 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 7.8 MiB/s wr, 739 op/s 2024-03-21T12:36:34.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:34 smithi189 ceph-mon[27080]: pgmap v281: 97 pgs: 97 active+clean; 305 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 7.8 MiB/s wr, 739 op/s 2024-03-21T12:36:34.821 INFO:teuthology.orchestra.run.smithi022.stdout: 5 59950 18.12 MB/sec warmup 94 sec latency 55.810 ms 2024-03-21T12:36:35.821 INFO:teuthology.orchestra.run.smithi022.stdout: 5 60574 18.16 MB/sec warmup 95 sec latency 28.729 ms 2024-03-21T12:36:36.108 INFO:teuthology.orchestra.run.smithi022.stdout:true 2024-03-21T12:36:36.255 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:36 smithi022 ceph-mon[24777]: pgmap v282: 97 pgs: 97 active+clean; 308 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 10 MiB/s wr, 1.02k op/s 2024-03-21T12:36:36.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:36 smithi189 ceph-mon[27080]: pgmap v282: 97 pgs: 97 active+clean; 308 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 10 MiB/s wr, 1.02k op/s 2024-03-21T12:36:36.821 INFO:teuthology.orchestra.run.smithi022.stdout: 5 60946 18.05 MB/sec warmup 96 sec latency 31.946 ms 2024-03-21T12:36:37.073 INFO:teuthology.orchestra.run.smithi022.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-21T12:36:37.073 INFO:teuthology.orchestra.run.smithi022.stdout:alertmanager.smithi022 smithi022 *:9093,9094 running (6m) 98s ago 8m 24.2M - ba2b418f427c 8fbe5dbdce93 2024-03-21T12:36:37.073 INFO:teuthology.orchestra.run.smithi022.stdout:crash.smithi022 smithi022 running (7m) 98s ago 7m 6405k - 17.2.0 e1d6a67b021e 47deb70fd250 2024-03-21T12:36:37.073 INFO:teuthology.orchestra.run.smithi022.stdout:crash.smithi189 smithi189 running (7m) 100s ago 7m 6405k - 17.2.0 e1d6a67b021e 9a2e0e9b1809 2024-03-21T12:36:37.073 INFO:teuthology.orchestra.run.smithi022.stdout:grafana.smithi022 smithi022 *:3000 running (6m) 98s ago 7m 52.7M - 8.3.5 dad864ee21e9 3c9ff131fc6b 2024-03-21T12:36:37.073 INFO:teuthology.orchestra.run.smithi022.stdout:mds.foofs.smithi022.bcnxho smithi022 running (2m) 98s ago 2m 13.2M - 17.2.0 e1d6a67b021e db26b67b8ee7 2024-03-21T12:36:37.073 INFO:teuthology.orchestra.run.smithi022.stdout:mds.foofs.smithi189.lluvko smithi189 running (2m) 100s ago 2m 14.5M - 17.2.0 e1d6a67b021e cf153aff1387 2024-03-21T12:36:37.073 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.smithi022.fektzs smithi022 *:9283 running (9m) 98s ago 9m 459M - 17.2.0 e1d6a67b021e ca06c9c2fb41 2024-03-21T12:36:37.073 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.smithi189.cnxzvu smithi189 *:8443,9283 running (7m) 100s ago 7m 413M - 17.2.0 e1d6a67b021e 5ea3c42ad934 2024-03-21T12:36:37.073 INFO:teuthology.orchestra.run.smithi022.stdout:mon.smithi022 smithi022 running (9m) 98s ago 9m 47.1M 2048M 17.2.0 e1d6a67b021e 0d248775e0e5 2024-03-21T12:36:37.074 INFO:teuthology.orchestra.run.smithi022.stdout:mon.smithi189 smithi189 running (7m) 100s ago 7m 37.2M 2048M 17.2.0 e1d6a67b021e 82da5f3cd49a 2024-03-21T12:36:37.074 INFO:teuthology.orchestra.run.smithi022.stdout:nfs.foo.0.0.smithi022.ynhbbq smithi022 *:2049 running (109s) 98s ago 108s 49.3M - 4.0 e1d6a67b021e 33e48e64c60b 2024-03-21T12:36:37.074 INFO:teuthology.orchestra.run.smithi022.stdout:nfs.foo.1.0.smithi189.yzsqml smithi189 *:2049 running (103s) 100s ago 103s 48.6M - 4.0 e1d6a67b021e 48fab2cbdde2 2024-03-21T12:36:37.074 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.smithi022 smithi022 *:9100 running (7m) 98s ago 7m 17.4M - 1dbe0e931976 cea183555968 2024-03-21T12:36:37.074 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.smithi189 smithi189 *:9100 running (7m) 100s ago 7m 19.5M - 1dbe0e931976 5db95224aff9 2024-03-21T12:36:37.074 INFO:teuthology.orchestra.run.smithi022.stdout:osd.0 smithi022 running (6m) 98s ago 6m 42.8M 3217M 17.2.0 e1d6a67b021e 54e6631bd10f 2024-03-21T12:36:37.074 INFO:teuthology.orchestra.run.smithi022.stdout:osd.1 smithi022 running (6m) 98s ago 6m 42.7M 3217M 17.2.0 e1d6a67b021e 68165c5d09cb 2024-03-21T12:36:37.074 INFO:teuthology.orchestra.run.smithi022.stdout:osd.2 smithi022 running (5m) 98s ago 5m 41.1M 3217M 17.2.0 e1d6a67b021e fbec9bef8433 2024-03-21T12:36:37.074 INFO:teuthology.orchestra.run.smithi022.stdout:osd.3 smithi022 running (5m) 98s ago 5m 40.3M 3217M 17.2.0 e1d6a67b021e b1462ffc1b9e 2024-03-21T12:36:37.074 INFO:teuthology.orchestra.run.smithi022.stdout:osd.4 smithi189 running (4m) 100s ago 4m 39.3M 3984M 17.2.0 e1d6a67b021e 1bedc4eb02ed 2024-03-21T12:36:37.074 INFO:teuthology.orchestra.run.smithi022.stdout:osd.5 smithi189 running (4m) 100s ago 4m 38.6M 3984M 17.2.0 e1d6a67b021e 25fbab9786a5 2024-03-21T12:36:37.074 INFO:teuthology.orchestra.run.smithi022.stdout:osd.6 smithi189 running (3m) 100s ago 3m 39.1M 3984M 17.2.0 e1d6a67b021e 0c192fcfd65b 2024-03-21T12:36:37.074 INFO:teuthology.orchestra.run.smithi022.stdout:osd.7 smithi189 running (3m) 100s ago 3m 43.4M 3984M 17.2.0 e1d6a67b021e 3d0723e1f863 2024-03-21T12:36:37.074 INFO:teuthology.orchestra.run.smithi022.stdout:prometheus.smithi022 smithi022 *:9095 running (6m) 98s ago 6m 63.3M - 514e6a882f6e 20a0eef27ec8 2024-03-21T12:36:37.274 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:37 smithi022 ceph-mon[24777]: from='client.24435 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:36:37.274 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:37 smithi022 ceph-mon[24777]: from='client.14654 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:36:37.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:37 smithi189 ceph-mon[27080]: from='client.24435 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:36:37.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:37 smithi189 ceph-mon[27080]: from='client.14654 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:36:37.616 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-03-21T12:36:37.616 INFO:teuthology.orchestra.run.smithi022.stdout: "mon": { 2024-03-21T12:36:37.616 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-21T12:36:37.616 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:36:37.616 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr": { 2024-03-21T12:36:37.616 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-21T12:36:37.616 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:36:37.616 INFO:teuthology.orchestra.run.smithi022.stdout: "osd": { 2024-03-21T12:36:37.616 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-03-21T12:36:37.616 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:36:37.616 INFO:teuthology.orchestra.run.smithi022.stdout: "mds": { 2024-03-21T12:36:37.617 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-21T12:36:37.617 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:36:37.617 INFO:teuthology.orchestra.run.smithi022.stdout: "overall": { 2024-03-21T12:36:37.617 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-03-21T12:36:37.617 INFO:teuthology.orchestra.run.smithi022.stdout: } 2024-03-21T12:36:37.617 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-03-21T12:36:37.821 INFO:teuthology.orchestra.run.smithi022.stdout: 5 61389 17.93 MB/sec warmup 97 sec latency 110.161 ms 2024-03-21T12:36:38.020 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-03-21T12:36:38.020 INFO:teuthology.orchestra.run.smithi022.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93", 2024-03-21T12:36:38.021 INFO:teuthology.orchestra.run.smithi022.stdout: "in_progress": true, 2024-03-21T12:36:38.021 INFO:teuthology.orchestra.run.smithi022.stdout: "services_complete": [], 2024-03-21T12:36:38.021 INFO:teuthology.orchestra.run.smithi022.stdout: "progress": "0/23 daemons upgraded", 2024-03-21T12:36:38.021 INFO:teuthology.orchestra.run.smithi022.stdout: "message": "Pulling quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93 image on host smithi189" 2024-03-21T12:36:38.021 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-03-21T12:36:38.289 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:38 smithi022 ceph-mon[24777]: pgmap v283: 97 pgs: 97 active+clean; 309 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 7.9 MiB/s wr, 715 op/s 2024-03-21T12:36:38.289 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:38 smithi022 ceph-mon[24777]: from='client.14658 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:36:38.289 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:38 smithi022 ceph-mon[24777]: from='client.? 172.21.15.22:0/1220424636' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:36:38.528 INFO:teuthology.orchestra.run.smithi022.stdout:HEALTH_OK 2024-03-21T12:36:38.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:38 smithi189 ceph-mon[27080]: pgmap v283: 97 pgs: 97 active+clean; 309 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 7.9 MiB/s wr, 715 op/s 2024-03-21T12:36:38.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:38 smithi189 ceph-mon[27080]: from='client.14658 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:36:38.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:38 smithi189 ceph-mon[27080]: from='client.? 172.21.15.22:0/1220424636' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:36:38.821 INFO:teuthology.orchestra.run.smithi022.stdout: 5 61755 17.84 MB/sec warmup 98 sec latency 76.456 ms 2024-03-21T12:36:39.317 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:39 smithi022 ceph-mon[24777]: from='client.24447 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:36:39.317 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:39 smithi022 ceph-mon[24777]: from='client.? 172.21.15.22:0/409736704' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-21T12:36:39.318 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:39 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:36:39.318 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:39 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi189.cnxzvu", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-21T12:36:39.318 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:39 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-21T12:36:39.318 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:39 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:36:39.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:39 smithi189 ceph-mon[27080]: from='client.24447 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:36:39.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:39 smithi189 ceph-mon[27080]: from='client.? 172.21.15.22:0/409736704' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-21T12:36:39.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:39 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:36:39.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:39 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi189.cnxzvu", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-21T12:36:39.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:39 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-21T12:36:39.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:39 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:36:39.821 INFO:teuthology.orchestra.run.smithi022.stdout: 5 62333 17.82 MB/sec warmup 99 sec latency 30.419 ms 2024-03-21T12:36:40.339 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:40 smithi022 ceph-mon[24777]: pgmap v284: 97 pgs: 97 active+clean; 310 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 7.2 MiB/s wr, 681 op/s 2024-03-21T12:36:40.339 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:40 smithi022 ceph-mon[24777]: Upgrade: Updating mgr.smithi189.cnxzvu 2024-03-21T12:36:40.339 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:40 smithi022 ceph-mon[24777]: Deploying daemon mgr.smithi189.cnxzvu on smithi189 2024-03-21T12:36:40.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:40 smithi189 ceph-mon[27080]: pgmap v284: 97 pgs: 97 active+clean; 310 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 7.2 MiB/s wr, 681 op/s 2024-03-21T12:36:40.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:40 smithi189 ceph-mon[27080]: Upgrade: Updating mgr.smithi189.cnxzvu 2024-03-21T12:36:40.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:40 smithi189 ceph-mon[27080]: Deploying daemon mgr.smithi189.cnxzvu on smithi189 2024-03-21T12:36:40.821 INFO:teuthology.orchestra.run.smithi022.stdout: 5 63003 17.80 MB/sec warmup 100 sec latency 23.479 ms 2024-03-21T12:36:41.821 INFO:teuthology.orchestra.run.smithi022.stdout: 5 63535 17.83 MB/sec warmup 101 sec latency 36.954 ms 2024-03-21T12:36:42.359 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:42 smithi022 ceph-mon[24777]: pgmap v285: 97 pgs: 97 active+clean; 318 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 9.0 KiB/s rd, 9.0 MiB/s wr, 839 op/s 2024-03-21T12:36:42.435 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:42 smithi189 ceph-mon[27080]: pgmap v285: 97 pgs: 97 active+clean; 318 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 9.0 KiB/s rd, 9.0 MiB/s wr, 839 op/s 2024-03-21T12:36:42.821 INFO:teuthology.orchestra.run.smithi022.stdout: 5 64146 17.86 MB/sec warmup 102 sec latency 26.540 ms 2024-03-21T12:36:43.823 INFO:teuthology.orchestra.run.smithi022.stdout: 5 64491 17.75 MB/sec warmup 103 sec latency 30.849 ms 2024-03-21T12:36:44.125 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:44 smithi022 ceph-mon[24777]: pgmap v286: 97 pgs: 97 active+clean; 318 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 9.0 KiB/s rd, 6.3 MiB/s wr, 619 op/s 2024-03-21T12:36:44.126 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:44 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:36:44.126 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:44 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:36:44.126 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:44 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:36:44.126 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:44 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:36:44.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:44 smithi189 ceph-mon[27080]: pgmap v286: 97 pgs: 97 active+clean; 318 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 9.0 KiB/s rd, 6.3 MiB/s wr, 619 op/s 2024-03-21T12:36:44.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:44 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:36:44.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:44 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:36:44.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:44 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:36:44.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:44 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:36:44.823 INFO:teuthology.orchestra.run.smithi022.stdout: 5 65109 17.68 MB/sec warmup 104 sec latency 41.664 ms 2024-03-21T12:36:45.823 INFO:teuthology.orchestra.run.smithi022.stdout: 5 65749 17.68 MB/sec warmup 105 sec latency 28.484 ms 2024-03-21T12:36:46.068 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:46 smithi189 ceph-mon[27080]: pgmap v287: 97 pgs: 97 active+clean; 331 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.0 MiB/s wr, 902 op/s 2024-03-21T12:36:46.135 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:46 smithi022 ceph-mon[24777]: pgmap v287: 97 pgs: 97 active+clean; 331 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.0 MiB/s wr, 902 op/s 2024-03-21T12:36:46.823 INFO:teuthology.orchestra.run.smithi022.stdout: 5 66506 17.66 MB/sec warmup 106 sec latency 32.777 ms 2024-03-21T12:36:47.823 INFO:teuthology.orchestra.run.smithi022.stdout: 5 67026 17.71 MB/sec warmup 107 sec latency 120.946 ms 2024-03-21T12:36:48.642 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:48 smithi022 ceph-mon[24777]: pgmap v288: 97 pgs: 97 active+clean; 332 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 7.6 MiB/s wr, 660 op/s 2024-03-21T12:36:48.642 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:48 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:36:48.642 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:48 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:36:48.672 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:48 smithi189 ceph-mon[27080]: pgmap v288: 97 pgs: 97 active+clean; 332 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 7.6 MiB/s wr, 660 op/s 2024-03-21T12:36:48.672 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:48 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:36:48.672 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:48 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:36:48.823 INFO:teuthology.orchestra.run.smithi022.stdout: 5 67457 17.66 MB/sec warmup 108 sec latency 31.044 ms 2024-03-21T12:36:49.823 INFO:teuthology.orchestra.run.smithi022.stdout: 5 67858 17.61 MB/sec warmup 109 sec latency 37.539 ms 2024-03-21T12:36:50.426 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:50 smithi022 ceph-mon[24777]: pgmap v289: 97 pgs: 97 active+clean; 333 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 7.0 MiB/s wr, 653 op/s 2024-03-21T12:36:50.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:50 smithi189 ceph-mon[27080]: pgmap v289: 97 pgs: 97 active+clean; 333 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 7.0 MiB/s wr, 653 op/s 2024-03-21T12:36:50.823 INFO:teuthology.orchestra.run.smithi022.stdout: 5 68211 17.53 MB/sec warmup 110 sec latency 39.339 ms 2024-03-21T12:36:51.823 INFO:teuthology.orchestra.run.smithi022.stdout: 5 68759 17.43 MB/sec warmup 111 sec latency 30.577 ms 2024-03-21T12:36:52.459 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:52 smithi022 ceph-mon[24777]: pgmap v290: 97 pgs: 97 active+clean; 343 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 9.1 KiB/s rd, 9.5 MiB/s wr, 999 op/s 2024-03-21T12:36:52.459 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:52 smithi022 ceph-mon[24777]: Standby manager daemon smithi189.cnxzvu restarted 2024-03-21T12:36:52.459 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:52 smithi022 ceph-mon[24777]: Standby manager daemon smithi189.cnxzvu started 2024-03-21T12:36:52.459 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:52 smithi022 ceph-mon[24777]: from='mgr.? 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi189.cnxzvu/crt"}]: dispatch 2024-03-21T12:36:52.459 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:52 smithi022 ceph-mon[24777]: from='mgr.? 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-21T12:36:52.459 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:52 smithi022 ceph-mon[24777]: from='mgr.? 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi189.cnxzvu/key"}]: dispatch 2024-03-21T12:36:52.459 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:52 smithi022 ceph-mon[24777]: from='mgr.? 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-21T12:36:52.460 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:52 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:36:52.460 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:52 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:36:52.460 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:52 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:36:52.460 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:52 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:36:52.460 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:52 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mgr fail", "who": "smithi022.fektzs"}]: dispatch 2024-03-21T12:36:52.460 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:52 smithi022 ceph-mon[24777]: osdmap e71: 8 total, 8 up, 8 in 2024-03-21T12:36:52.641 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:52 smithi189 ceph-mon[27080]: pgmap v290: 97 pgs: 97 active+clean; 343 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 9.1 KiB/s rd, 9.5 MiB/s wr, 999 op/s 2024-03-21T12:36:52.642 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:52 smithi189 ceph-mon[27080]: Standby manager daemon smithi189.cnxzvu restarted 2024-03-21T12:36:52.642 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:52 smithi189 ceph-mon[27080]: Standby manager daemon smithi189.cnxzvu started 2024-03-21T12:36:52.642 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:52 smithi189 ceph-mon[27080]: from='mgr.? 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi189.cnxzvu/crt"}]: dispatch 2024-03-21T12:36:52.642 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:52 smithi189 ceph-mon[27080]: from='mgr.? 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-21T12:36:52.642 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:52 smithi189 ceph-mon[27080]: from='mgr.? 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi189.cnxzvu/key"}]: dispatch 2024-03-21T12:36:52.642 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:52 smithi189 ceph-mon[27080]: from='mgr.? 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-21T12:36:52.642 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:52 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:36:52.642 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:52 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:36:52.642 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:52 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' 2024-03-21T12:36:52.642 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:52 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:36:52.642 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:52 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mgr fail", "who": "smithi022.fektzs"}]: dispatch 2024-03-21T12:36:52.642 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:52 smithi189 ceph-mon[27080]: osdmap e71: 8 total, 8 up, 8 in 2024-03-21T12:36:52.823 INFO:teuthology.orchestra.run.smithi022.stdout: 5 69345 17.45 MB/sec warmup 112 sec latency 30.851 ms 2024-03-21T12:36:53.464 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:53 smithi022 ceph-mon[24777]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "mon metadata", "id": "smithi022"}]: dispatch 2024-03-21T12:36:53.464 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:53 smithi022 ceph-mon[24777]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "mon metadata", "id": "smithi189"}]: dispatch 2024-03-21T12:36:53.464 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:53 smithi022 ceph-mon[24777]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi022.bcnxho"}]: dispatch 2024-03-21T12:36:53.464 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:53 smithi022 ceph-mon[24777]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi189.lluvko"}]: dispatch 2024-03-21T12:36:53.464 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:53 smithi022 ceph-mon[24777]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "mgr metadata", "who": "smithi189.cnxzvu", "id": "smithi189.cnxzvu"}]: dispatch 2024-03-21T12:36:53.464 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:53 smithi022 ceph-mon[24777]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-21T12:36:53.465 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:53 smithi022 ceph-mon[24777]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-21T12:36:53.465 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:53 smithi022 ceph-mon[24777]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-21T12:36:53.465 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:53 smithi022 ceph-mon[24777]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-21T12:36:53.465 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:53 smithi022 ceph-mon[24777]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-21T12:36:53.465 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:53 smithi022 ceph-mon[24777]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-21T12:36:53.465 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:53 smithi022 ceph-mon[24777]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-21T12:36:53.465 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:53 smithi022 ceph-mon[24777]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-21T12:36:53.465 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:53 smithi022 ceph-mon[24777]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "mgr fail", "who": "smithi022.fektzs"}]': finished 2024-03-21T12:36:53.465 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:53 smithi022 ceph-mon[24777]: mgrmap e22: smithi189.cnxzvu(active, starting, since 0.305357s) 2024-03-21T12:36:53.465 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:53 smithi022 ceph-mon[24777]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-21T12:36:53.465 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:53 smithi022 ceph-mon[24777]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-21T12:36:53.465 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:53 smithi022 ceph-mon[24777]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-21T12:36:53.465 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:53 smithi022 ceph-mon[24777]: Manager daemon smithi189.cnxzvu is now available 2024-03-21T12:36:53.465 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:53 smithi022 ceph-mon[24777]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:36:53.465 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:53 smithi022 ceph-mon[24777]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:36:53.466 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:53 smithi022 ceph-mon[24777]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:36:53.466 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:53 smithi022 ceph-mon[24777]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi189.cnxzvu/mirror_snapshot_schedule"}]: dispatch 2024-03-21T12:36:53.466 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:53 smithi022 ceph-mon[24777]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi189.cnxzvu/mirror_snapshot_schedule"}]: dispatch 2024-03-21T12:36:53.466 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:53 smithi022 ceph-mon[24777]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi189.cnxzvu/trash_purge_schedule"}]: dispatch 2024-03-21T12:36:53.466 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:53 smithi022 ceph-mon[24777]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi189.cnxzvu/trash_purge_schedule"}]: dispatch 2024-03-21T12:36:53.466 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:53 smithi022 ceph-mon[24777]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:36:53.466 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:53 smithi022 ceph-mon[24777]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:36:53.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:53 smithi189 ceph-mon[27080]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "mon metadata", "id": "smithi022"}]: dispatch 2024-03-21T12:36:53.787 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:53 smithi189 ceph-mon[27080]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "mon metadata", "id": "smithi189"}]: dispatch 2024-03-21T12:36:53.787 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:53 smithi189 ceph-mon[27080]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi022.bcnxho"}]: dispatch 2024-03-21T12:36:53.787 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:53 smithi189 ceph-mon[27080]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi189.lluvko"}]: dispatch 2024-03-21T12:36:53.787 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:53 smithi189 ceph-mon[27080]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "mgr metadata", "who": "smithi189.cnxzvu", "id": "smithi189.cnxzvu"}]: dispatch 2024-03-21T12:36:53.787 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:53 smithi189 ceph-mon[27080]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-21T12:36:53.787 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:53 smithi189 ceph-mon[27080]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-21T12:36:53.787 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:53 smithi189 ceph-mon[27080]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-21T12:36:53.787 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:53 smithi189 ceph-mon[27080]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-21T12:36:53.787 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:53 smithi189 ceph-mon[27080]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-21T12:36:53.787 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:53 smithi189 ceph-mon[27080]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-21T12:36:53.787 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:53 smithi189 ceph-mon[27080]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-21T12:36:53.787 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:53 smithi189 ceph-mon[27080]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-21T12:36:53.787 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:53 smithi189 ceph-mon[27080]: from='mgr.14202 172.21.15.22:0/175777495' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "mgr fail", "who": "smithi022.fektzs"}]': finished 2024-03-21T12:36:53.788 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:53 smithi189 ceph-mon[27080]: mgrmap e22: smithi189.cnxzvu(active, starting, since 0.305357s) 2024-03-21T12:36:53.788 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:53 smithi189 ceph-mon[27080]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-21T12:36:53.788 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:53 smithi189 ceph-mon[27080]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-21T12:36:53.788 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:53 smithi189 ceph-mon[27080]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-21T12:36:53.788 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:53 smithi189 ceph-mon[27080]: Manager daemon smithi189.cnxzvu is now available 2024-03-21T12:36:53.788 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:53 smithi189 ceph-mon[27080]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:36:53.788 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:53 smithi189 ceph-mon[27080]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:36:53.788 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:53 smithi189 ceph-mon[27080]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:36:53.788 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:53 smithi189 ceph-mon[27080]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi189.cnxzvu/mirror_snapshot_schedule"}]: dispatch 2024-03-21T12:36:53.788 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:53 smithi189 ceph-mon[27080]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi189.cnxzvu/mirror_snapshot_schedule"}]: dispatch 2024-03-21T12:36:53.788 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:53 smithi189 ceph-mon[27080]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi189.cnxzvu/trash_purge_schedule"}]: dispatch 2024-03-21T12:36:53.789 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:53 smithi189 ceph-mon[27080]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi189.cnxzvu/trash_purge_schedule"}]: dispatch 2024-03-21T12:36:53.789 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:53 smithi189 ceph-mon[27080]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:36:53.789 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:53 smithi189 ceph-mon[27080]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:36:53.823 INFO:teuthology.orchestra.run.smithi022.stdout: 5 70005 17.40 MB/sec warmup 113 sec latency 32.567 ms 2024-03-21T12:36:54.471 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:54 smithi022 ceph-mon[24777]: Deploying cephadm binary to smithi189 2024-03-21T12:36:54.471 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:54 smithi022 ceph-mon[24777]: Deploying cephadm binary to smithi022 2024-03-21T12:36:54.471 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:54 smithi022 ceph-mon[24777]: mgrmap e23: smithi189.cnxzvu(active, since 1.32354s) 2024-03-21T12:36:54.472 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:54 smithi022 ceph-mon[24777]: pgmap v3: 97 pgs: 97 active+clean; 337 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-03-21T12:36:54.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:54 smithi189 ceph-mon[27080]: Deploying cephadm binary to smithi189 2024-03-21T12:36:54.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:54 smithi189 ceph-mon[27080]: Deploying cephadm binary to smithi022 2024-03-21T12:36:54.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:54 smithi189 ceph-mon[27080]: mgrmap e23: smithi189.cnxzvu(active, since 1.32354s) 2024-03-21T12:36:54.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:54 smithi189 ceph-mon[27080]: pgmap v3: 97 pgs: 97 active+clean; 337 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-03-21T12:36:54.823 INFO:teuthology.orchestra.run.smithi022.stdout: 5 70640 17.49 MB/sec warmup 114 sec latency 24.395 ms 2024-03-21T12:36:55.475 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:55 smithi022 ceph-mon[24777]: [21/Mar/2024:12:36:54] ENGINE Bus STARTING 2024-03-21T12:36:55.475 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:55 smithi022 ceph-mon[24777]: [21/Mar/2024:12:36:54] ENGINE Serving on https://172.21.15.189:7150 2024-03-21T12:36:55.476 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:55 smithi022 ceph-mon[24777]: [21/Mar/2024:12:36:54] ENGINE Serving on http://172.21.15.189:8765 2024-03-21T12:36:55.476 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:55 smithi022 ceph-mon[24777]: [21/Mar/2024:12:36:54] ENGINE Bus STARTED 2024-03-21T12:36:55.476 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:55 smithi022 ceph-mon[24777]: pgmap v4: 97 pgs: 97 active+clean; 337 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-03-21T12:36:55.476 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:55 smithi022 ceph-mon[24777]: mgrmap e24: smithi189.cnxzvu(active, since 2s) 2024-03-21T12:36:55.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:55 smithi189 ceph-mon[27080]: [21/Mar/2024:12:36:54] ENGINE Bus STARTING 2024-03-21T12:36:55.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:55 smithi189 ceph-mon[27080]: [21/Mar/2024:12:36:54] ENGINE Serving on https://172.21.15.189:7150 2024-03-21T12:36:55.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:55 smithi189 ceph-mon[27080]: [21/Mar/2024:12:36:54] ENGINE Serving on http://172.21.15.189:8765 2024-03-21T12:36:55.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:55 smithi189 ceph-mon[27080]: [21/Mar/2024:12:36:54] ENGINE Bus STARTED 2024-03-21T12:36:55.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:55 smithi189 ceph-mon[27080]: pgmap v4: 97 pgs: 97 active+clean; 337 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-03-21T12:36:55.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:55 smithi189 ceph-mon[27080]: mgrmap e24: smithi189.cnxzvu(active, since 2s) 2024-03-21T12:36:55.823 INFO:teuthology.orchestra.run.smithi022.stdout: 5 71304 17.51 MB/sec warmup 115 sec latency 32.387 ms 2024-03-21T12:36:56.823 INFO:teuthology.orchestra.run.smithi022.stdout: 5 71740 17.45 MB/sec warmup 116 sec latency 27.959 ms 2024-03-21T12:36:57.496 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:57 smithi022 ceph-mon[24777]: pgmap v5: 97 pgs: 97 active+clean; 337 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-03-21T12:36:57.496 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:57 smithi022 ceph-mon[24777]: mgrmap e25: smithi189.cnxzvu(active, since 4s) 2024-03-21T12:36:57.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:57 smithi189 ceph-mon[27080]: pgmap v5: 97 pgs: 97 active+clean; 337 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-03-21T12:36:57.787 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:57 smithi189 ceph-mon[27080]: mgrmap e25: smithi189.cnxzvu(active, since 4s) 2024-03-21T12:36:57.823 INFO:teuthology.orchestra.run.smithi022.stdout: 5 72292 17.35 MB/sec warmup 117 sec latency 45.890 ms 2024-03-21T12:36:58.823 INFO:teuthology.orchestra.run.smithi022.stdout: 5 72767 17.30 MB/sec warmup 118 sec latency 34.236 ms 2024-03-21T12:36:59.528 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:36:59 smithi022 ceph-mon[24777]: pgmap v6: 97 pgs: 97 active+clean; 337 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-03-21T12:36:59.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:36:59 smithi189 ceph-mon[27080]: pgmap v6: 97 pgs: 97 active+clean; 337 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-03-21T12:36:59.824 INFO:teuthology.orchestra.run.smithi022.stdout: 5 73362 17.30 MB/sec warmup 119 sec latency 106.996 ms 2024-03-21T12:37:01.540 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:01 smithi022 ceph-mon[24777]: pgmap v7: 97 pgs: 97 active+clean; 341 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 37 KiB/s rd, 6.4 MiB/s wr, 604 op/s 2024-03-21T12:37:01.540 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:01 smithi022 ceph-mon[24777]: Standby manager daemon smithi022.fektzs started 2024-03-21T12:37:01.540 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:01 smithi022 ceph-mon[24777]: from='mgr.? 172.21.15.22:0/730035198' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi022.fektzs/crt"}]: dispatch 2024-03-21T12:37:01.541 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:01 smithi022 ceph-mon[24777]: from='mgr.? 172.21.15.22:0/730035198' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-21T12:37:01.541 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:01 smithi022 ceph-mon[24777]: from='mgr.? 172.21.15.22:0/730035198' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi022.fektzs/key"}]: dispatch 2024-03-21T12:37:01.541 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:01 smithi022 ceph-mon[24777]: from='mgr.? 172.21.15.22:0/730035198' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-21T12:37:01.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:01 smithi189 ceph-mon[27080]: pgmap v7: 97 pgs: 97 active+clean; 341 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 37 KiB/s rd, 6.4 MiB/s wr, 604 op/s 2024-03-21T12:37:01.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:01 smithi189 ceph-mon[27080]: Standby manager daemon smithi022.fektzs started 2024-03-21T12:37:01.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:01 smithi189 ceph-mon[27080]: from='mgr.? 172.21.15.22:0/730035198' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi022.fektzs/crt"}]: dispatch 2024-03-21T12:37:01.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:01 smithi189 ceph-mon[27080]: from='mgr.? 172.21.15.22:0/730035198' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-21T12:37:01.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:01 smithi189 ceph-mon[27080]: from='mgr.? 172.21.15.22:0/730035198' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi022.fektzs/key"}]: dispatch 2024-03-21T12:37:01.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:01 smithi189 ceph-mon[27080]: from='mgr.? 172.21.15.22:0/730035198' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-21T12:37:01.824 INFO:teuthology.orchestra.run.smithi022.stdout: 5 74482 23.23 MB/sec execute 1 sec latency 38.642 ms 2024-03-21T12:37:02.449 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:02 smithi189 ceph-mon[27080]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "mgr metadata", "who": "smithi022.fektzs", "id": "smithi022.fektzs"}]: dispatch 2024-03-21T12:37:02.450 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:02 smithi189 ceph-mon[27080]: mgrmap e26: smithi189.cnxzvu(active, since 9s), standbys: smithi022.fektzs 2024-03-21T12:37:02.554 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:02 smithi022 ceph-mon[24777]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "mgr metadata", "who": "smithi022.fektzs", "id": "smithi022.fektzs"}]: dispatch 2024-03-21T12:37:02.554 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:02 smithi022 ceph-mon[24777]: mgrmap e26: smithi189.cnxzvu(active, since 9s), standbys: smithi022.fektzs 2024-03-21T12:37:02.824 INFO:teuthology.orchestra.run.smithi022.stdout: 5 75070 21.49 MB/sec execute 2 sec latency 23.768 ms 2024-03-21T12:37:03.472 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:03 smithi022 ceph-mon[24777]: pgmap v8: 97 pgs: 97 active+clean; 341 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 5.0 MiB/s wr, 469 op/s 2024-03-21T12:37:03.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:03 smithi189 ceph-mon[27080]: pgmap v8: 97 pgs: 97 active+clean; 341 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 5.0 MiB/s wr, 469 op/s 2024-03-21T12:37:03.824 INFO:teuthology.orchestra.run.smithi022.stdout: 5 75491 16.34 MB/sec execute 3 sec latency 34.981 ms 2024-03-21T12:37:04.824 INFO:teuthology.orchestra.run.smithi022.stdout: 5 75949 14.03 MB/sec execute 4 sec latency 64.316 ms 2024-03-21T12:37:05.578 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:05 smithi022 ceph-mon[24777]: pgmap v9: 97 pgs: 97 active+clean; 343 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 28 KiB/s rd, 7.9 MiB/s wr, 722 op/s 2024-03-21T12:37:05.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:05 smithi189 ceph-mon[27080]: pgmap v9: 97 pgs: 97 active+clean; 343 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 28 KiB/s rd, 7.9 MiB/s wr, 722 op/s 2024-03-21T12:37:05.824 INFO:teuthology.orchestra.run.smithi022.stdout: 5 76507 15.27 MB/sec execute 5 sec latency 64.523 ms 2024-03-21T12:37:06.824 INFO:teuthology.orchestra.run.smithi022.stdout: 5 77131 14.79 MB/sec execute 6 sec latency 37.592 ms 2024-03-21T12:37:06.838 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:06 smithi022 ceph-mon[24777]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:37:06.838 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:06 smithi022 ceph-mon[24777]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:37:07.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:06 smithi189 ceph-mon[27080]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:37:07.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:06 smithi189 ceph-mon[27080]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:37:07.828 INFO:teuthology.orchestra.run.smithi022.stdout: 5 77557 14.48 MB/sec execute 7 sec latency 34.457 ms 2024-03-21T12:37:07.847 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:07 smithi022 ceph-mon[24777]: pgmap v10: 97 pgs: 97 active+clean; 343 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 7.2 MiB/s wr, 660 op/s 2024-03-21T12:37:07.847 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:07 smithi022 ceph-mon[24777]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:37:07.847 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:07 smithi022 ceph-mon[24777]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:37:07.848 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:07 smithi022 ceph-mon[24777]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:37:08.107 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:07 smithi189 ceph-mon[27080]: pgmap v10: 97 pgs: 97 active+clean; 343 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 7.2 MiB/s wr, 660 op/s 2024-03-21T12:37:08.107 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:07 smithi189 ceph-mon[27080]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:37:08.107 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:07 smithi189 ceph-mon[27080]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:37:08.107 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:07 smithi189 ceph-mon[27080]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:37:08.828 INFO:teuthology.orchestra.run.smithi022.stdout: 5 78282 15.82 MB/sec execute 8 sec latency 30.176 ms 2024-03-21T12:37:09.141 INFO:teuthology.orchestra.run.smithi022.stdout:true 2024-03-21T12:37:09.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:09 smithi189 ceph-mon[27080]: pgmap v11: 97 pgs: 97 active+clean; 343 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 7.2 MiB/s wr, 660 op/s 2024-03-21T12:37:09.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:09 smithi189 ceph-mon[27080]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:37:09.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:09 smithi189 ceph-mon[27080]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:37:09.789 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:09 smithi022 ceph-mon[24777]: pgmap v11: 97 pgs: 97 active+clean; 343 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 7.2 MiB/s wr, 660 op/s 2024-03-21T12:37:09.789 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:09 smithi022 ceph-mon[24777]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:37:09.790 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:09 smithi022 ceph-mon[24777]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:37:09.829 INFO:teuthology.orchestra.run.smithi022.stdout: 5 78710 15.76 MB/sec execute 9 sec latency 114.616 ms 2024-03-21T12:37:10.201 INFO:teuthology.orchestra.run.smithi022.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-21T12:37:10.202 INFO:teuthology.orchestra.run.smithi022.stdout:alertmanager.smithi022 smithi022 *:9093,9094 running (7m) 1s ago 8m 27.7M - 0.23.0 ba2b418f427c 8fbe5dbdce93 2024-03-21T12:37:10.202 INFO:teuthology.orchestra.run.smithi022.stdout:crash.smithi022 smithi022 running (8m) 1s ago 8m 6405k - 17.2.0 e1d6a67b021e 47deb70fd250 2024-03-21T12:37:10.202 INFO:teuthology.orchestra.run.smithi022.stdout:crash.smithi189 smithi189 running (7m) 4s ago 7m 6405k - 17.2.0 e1d6a67b021e 9a2e0e9b1809 2024-03-21T12:37:10.202 INFO:teuthology.orchestra.run.smithi022.stdout:grafana.smithi022 smithi022 *:3000 running (6m) 1s ago 8m 55.2M - 8.3.5 dad864ee21e9 3c9ff131fc6b 2024-03-21T12:37:10.202 INFO:teuthology.orchestra.run.smithi022.stdout:mds.foofs.smithi022.bcnxho smithi022 running (2m) 1s ago 2m 36.7M - 17.2.0 e1d6a67b021e db26b67b8ee7 2024-03-21T12:37:10.202 INFO:teuthology.orchestra.run.smithi022.stdout:mds.foofs.smithi189.lluvko smithi189 running (2m) 4s ago 2m 15.3M - 17.2.0 e1d6a67b021e cf153aff1387 2024-03-21T12:37:10.202 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.smithi022.fektzs smithi022 *:9283 running (9m) 1s ago 9m 346M - 17.2.0 e1d6a67b021e ca06c9c2fb41 2024-03-21T12:37:10.202 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.smithi189.cnxzvu smithi189 *:8443,9283 running (27s) 4s ago 7m 488M - 19.0.0-2267-g960fbbe4 19e12293512f 07f3cf369ff7 2024-03-21T12:37:10.202 INFO:teuthology.orchestra.run.smithi022.stdout:mon.smithi022 smithi022 running (9m) 1s ago 10m 49.9M 2048M 17.2.0 e1d6a67b021e 0d248775e0e5 2024-03-21T12:37:10.202 INFO:teuthology.orchestra.run.smithi022.stdout:mon.smithi189 smithi189 running (7m) 4s ago 7m 40.1M 2048M 17.2.0 e1d6a67b021e 82da5f3cd49a 2024-03-21T12:37:10.202 INFO:teuthology.orchestra.run.smithi022.stdout:nfs.foo.0.0.smithi022.ynhbbq smithi022 *:2049 running (2m) 1s ago 2m 250M - 4.0 e1d6a67b021e 33e48e64c60b 2024-03-21T12:37:10.202 INFO:teuthology.orchestra.run.smithi022.stdout:nfs.foo.1.0.smithi189.yzsqml smithi189 *:2049 running (2m) 4s ago 2m 73.3M - 4.0 e1d6a67b021e 48fab2cbdde2 2024-03-21T12:37:10.203 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.smithi022 smithi022 *:9100 running (8m) 1s ago 8m 17.5M - 1.3.1 1dbe0e931976 cea183555968 2024-03-21T12:37:10.203 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.smithi189 smithi189 *:9100 running (7m) 4s ago 7m 22.6M - 1.3.1 1dbe0e931976 5db95224aff9 2024-03-21T12:37:10.203 INFO:teuthology.orchestra.run.smithi022.stdout:osd.0 smithi022 running (7m) 1s ago 7m 294M 3217M 17.2.0 e1d6a67b021e 54e6631bd10f 2024-03-21T12:37:10.203 INFO:teuthology.orchestra.run.smithi022.stdout:osd.1 smithi022 running (6m) 1s ago 6m 233M 3217M 17.2.0 e1d6a67b021e 68165c5d09cb 2024-03-21T12:37:10.203 INFO:teuthology.orchestra.run.smithi022.stdout:osd.2 smithi022 running (6m) 1s ago 6m 184M 3217M 17.2.0 e1d6a67b021e fbec9bef8433 2024-03-21T12:37:10.203 INFO:teuthology.orchestra.run.smithi022.stdout:osd.3 smithi022 running (5m) 1s ago 5m 256M 3217M 17.2.0 e1d6a67b021e b1462ffc1b9e 2024-03-21T12:37:10.203 INFO:teuthology.orchestra.run.smithi022.stdout:osd.4 smithi189 running (5m) 4s ago 5m 276M 3984M 17.2.0 e1d6a67b021e 1bedc4eb02ed 2024-03-21T12:37:10.203 INFO:teuthology.orchestra.run.smithi022.stdout:osd.5 smithi189 running (4m) 4s ago 4m 313M 3984M 17.2.0 e1d6a67b021e 25fbab9786a5 2024-03-21T12:37:10.203 INFO:teuthology.orchestra.run.smithi022.stdout:osd.6 smithi189 running (4m) 4s ago 4m 262M 3984M 17.2.0 e1d6a67b021e 0c192fcfd65b 2024-03-21T12:37:10.203 INFO:teuthology.orchestra.run.smithi022.stdout:osd.7 smithi189 running (4m) 4s ago 4m 384M 3984M 17.2.0 e1d6a67b021e 3d0723e1f863 2024-03-21T12:37:10.203 INFO:teuthology.orchestra.run.smithi022.stdout:prometheus.smithi022 smithi022 *:9095 running (7m) 1s ago 7m 64.2M - 2.33.4 514e6a882f6e 20a0eef27ec8 2024-03-21T12:37:10.796 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:10 smithi022 ceph-mon[24777]: from='client.14700 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:37:10.797 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:10 smithi022 ceph-mon[24777]: from='client.24487 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:37:10.797 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:10 smithi022 ceph-mon[24777]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:37:10.797 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:10 smithi022 ceph-mon[24777]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:37:10.797 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:10 smithi022 ceph-mon[24777]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-21T12:37:10.797 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:10 smithi022 ceph-mon[24777]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-21T12:37:10.797 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:10 smithi022 ceph-mon[24777]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-21T12:37:10.797 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:10 smithi022 ceph-mon[24777]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-21T12:37:10.797 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:10 smithi022 ceph-mon[24777]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-21T12:37:10.797 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:10 smithi022 ceph-mon[24777]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-21T12:37:10.797 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:10 smithi022 ceph-mon[24777]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-03-21T12:37:10.797 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:10 smithi022 ceph-mon[24777]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-03-21T12:37:10.797 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:10 smithi022 ceph-mon[24777]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:37:10.829 INFO:teuthology.orchestra.run.smithi022.stdout: 5 79071 14.76 MB/sec execute 10 sec latency 90.532 ms 2024-03-21T12:37:10.914 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-03-21T12:37:10.914 INFO:teuthology.orchestra.run.smithi022.stdout: "mon": { 2024-03-21T12:37:10.914 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-21T12:37:10.914 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:37:10.915 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr": { 2024-03-21T12:37:10.915 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-03-21T12:37:10.915 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 1 2024-03-21T12:37:10.915 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:37:10.915 INFO:teuthology.orchestra.run.smithi022.stdout: "osd": { 2024-03-21T12:37:10.915 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-03-21T12:37:10.915 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:37:10.915 INFO:teuthology.orchestra.run.smithi022.stdout: "mds": { 2024-03-21T12:37:10.915 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-21T12:37:10.915 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:37:10.915 INFO:teuthology.orchestra.run.smithi022.stdout: "overall": { 2024-03-21T12:37:10.915 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-03-21T12:37:10.915 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 1 2024-03-21T12:37:10.915 INFO:teuthology.orchestra.run.smithi022.stdout: } 2024-03-21T12:37:10.915 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-03-21T12:37:10.930 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:10 smithi189 ceph-mon[27080]: from='client.14700 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:37:10.930 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:10 smithi189 ceph-mon[27080]: from='client.24487 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:37:10.930 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:10 smithi189 ceph-mon[27080]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:37:10.930 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:10 smithi189 ceph-mon[27080]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:37:10.930 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:10 smithi189 ceph-mon[27080]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-21T12:37:10.930 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:10 smithi189 ceph-mon[27080]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-21T12:37:10.931 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:10 smithi189 ceph-mon[27080]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-21T12:37:10.931 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:10 smithi189 ceph-mon[27080]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-21T12:37:10.931 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:10 smithi189 ceph-mon[27080]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-21T12:37:10.931 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:10 smithi189 ceph-mon[27080]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-21T12:37:10.931 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:10 smithi189 ceph-mon[27080]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-03-21T12:37:10.931 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:10 smithi189 ceph-mon[27080]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-03-21T12:37:10.931 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:10 smithi189 ceph-mon[27080]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:37:11.429 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-03-21T12:37:11.429 INFO:teuthology.orchestra.run.smithi022.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93", 2024-03-21T12:37:11.429 INFO:teuthology.orchestra.run.smithi022.stdout: "in_progress": true, 2024-03-21T12:37:11.430 INFO:teuthology.orchestra.run.smithi022.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-21T12:37:11.430 INFO:teuthology.orchestra.run.smithi022.stdout: "services_complete": [], 2024-03-21T12:37:11.430 INFO:teuthology.orchestra.run.smithi022.stdout: "progress": "1/23 daemons upgraded", 2024-03-21T12:37:11.430 INFO:teuthology.orchestra.run.smithi022.stdout: "message": "", 2024-03-21T12:37:11.430 INFO:teuthology.orchestra.run.smithi022.stdout: "is_paused": false 2024-03-21T12:37:11.430 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-03-21T12:37:11.654 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:11 smithi189 ceph-mon[27080]: from='client.24491 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:37:11.654 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:11 smithi189 ceph-mon[27080]: Adjusting osd_memory_target on smithi022 to 1937M 2024-03-21T12:37:11.654 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:11 smithi189 ceph-mon[27080]: pgmap v12: 97 pgs: 97 active+clean; 334 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 10 MiB/s wr, 979 op/s 2024-03-21T12:37:11.654 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:11 smithi189 ceph-mon[27080]: from='client.? 172.21.15.22:0/2625877804' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:37:11.654 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:11 smithi189 ceph-mon[27080]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:37:11.654 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:11 smithi189 ceph-mon[27080]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:37:11.816 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:11 smithi022 ceph-mon[24777]: from='client.24491 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:37:11.817 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:11 smithi022 ceph-mon[24777]: Adjusting osd_memory_target on smithi022 to 1937M 2024-03-21T12:37:11.817 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:11 smithi022 ceph-mon[24777]: pgmap v12: 97 pgs: 97 active+clean; 334 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 10 MiB/s wr, 979 op/s 2024-03-21T12:37:11.817 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:11 smithi022 ceph-mon[24777]: from='client.? 172.21.15.22:0/2625877804' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:37:11.817 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:11 smithi022 ceph-mon[24777]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:37:11.817 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:11 smithi022 ceph-mon[24777]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:37:11.817 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:11 smithi022 ceph-mon[24777]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-21T12:37:11.817 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:11 smithi022 ceph-mon[24777]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-21T12:37:11.817 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:11 smithi022 ceph-mon[24777]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-21T12:37:11.817 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:11 smithi022 ceph-mon[24777]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-21T12:37:11.817 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:11 smithi022 ceph-mon[24777]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-21T12:37:11.817 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:11 smithi022 ceph-mon[24777]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-21T12:37:11.817 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:11 smithi022 ceph-mon[24777]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-21T12:37:11.817 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:11 smithi022 ceph-mon[24777]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-21T12:37:11.818 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:11 smithi022 ceph-mon[24777]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:37:11.818 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:11 smithi022 ceph-mon[24777]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:37:11.818 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:11 smithi022 ceph-mon[24777]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:37:11.829 INFO:teuthology.orchestra.run.smithi022.stdout: 5 79541 13.84 MB/sec execute 11 sec latency 35.276 ms 2024-03-21T12:37:11.988 INFO:teuthology.orchestra.run.smithi022.stdout:HEALTH_OK 2024-03-21T12:37:12.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:11 smithi189 ceph-mon[27080]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-21T12:37:12.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:11 smithi189 ceph-mon[27080]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-21T12:37:12.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:11 smithi189 ceph-mon[27080]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-21T12:37:12.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:11 smithi189 ceph-mon[27080]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-21T12:37:12.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:11 smithi189 ceph-mon[27080]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-21T12:37:12.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:11 smithi189 ceph-mon[27080]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-21T12:37:12.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:11 smithi189 ceph-mon[27080]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-21T12:37:12.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:11 smithi189 ceph-mon[27080]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-21T12:37:12.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:11 smithi189 ceph-mon[27080]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:37:12.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:11 smithi189 ceph-mon[27080]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:37:12.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:11 smithi189 ceph-mon[27080]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:37:12.829 INFO:teuthology.orchestra.run.smithi022.stdout: 5 79906 13.72 MB/sec execute 12 sec latency 75.764 ms 2024-03-21T12:37:12.877 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:12 smithi022 ceph-mon[24777]: Adjusting osd_memory_target on smithi189 to 2704M 2024-03-21T12:37:12.877 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:12 smithi022 ceph-mon[24777]: Updating smithi022:/etc/ceph/ceph.conf 2024-03-21T12:37:12.878 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:12 smithi022 ceph-mon[24777]: Updating smithi189:/etc/ceph/ceph.conf 2024-03-21T12:37:12.878 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:12 smithi022 ceph-mon[24777]: from='client.14712 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:37:12.878 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:12 smithi022 ceph-mon[24777]: Updating smithi022:/var/lib/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/config/ceph.conf 2024-03-21T12:37:12.878 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:12 smithi022 ceph-mon[24777]: Updating smithi189:/var/lib/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/config/ceph.conf 2024-03-21T12:37:12.878 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:12 smithi022 ceph-mon[24777]: from='client.? 172.21.15.22:0/197470598' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-21T12:37:13.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:12 smithi189 ceph-mon[27080]: Adjusting osd_memory_target on smithi189 to 2704M 2024-03-21T12:37:13.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:12 smithi189 ceph-mon[27080]: Updating smithi022:/etc/ceph/ceph.conf 2024-03-21T12:37:13.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:12 smithi189 ceph-mon[27080]: Updating smithi189:/etc/ceph/ceph.conf 2024-03-21T12:37:13.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:12 smithi189 ceph-mon[27080]: from='client.14712 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:37:13.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:12 smithi189 ceph-mon[27080]: Updating smithi022:/var/lib/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/config/ceph.conf 2024-03-21T12:37:13.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:12 smithi189 ceph-mon[27080]: Updating smithi189:/var/lib/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/config/ceph.conf 2024-03-21T12:37:13.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:12 smithi189 ceph-mon[27080]: from='client.? 172.21.15.22:0/197470598' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-21T12:37:13.829 INFO:teuthology.orchestra.run.smithi022.stdout: 5 80471 13.82 MB/sec execute 13 sec latency 60.897 ms 2024-03-21T12:37:13.890 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:13 smithi022 ceph-mon[24777]: Updating smithi189:/etc/ceph/ceph.client.admin.keyring 2024-03-21T12:37:13.890 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:13 smithi022 ceph-mon[24777]: Updating smithi022:/etc/ceph/ceph.client.admin.keyring 2024-03-21T12:37:13.890 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:13 smithi022 ceph-mon[24777]: Updating smithi189:/var/lib/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/config/ceph.client.admin.keyring 2024-03-21T12:37:13.890 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:13 smithi022 ceph-mon[24777]: Updating smithi022:/var/lib/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/config/ceph.client.admin.keyring 2024-03-21T12:37:13.890 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:13 smithi022 ceph-mon[24777]: pgmap v13: 97 pgs: 97 active+clean; 334 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.7 MiB/s wr, 628 op/s 2024-03-21T12:37:13.890 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:13 smithi022 ceph-mon[24777]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:37:13.890 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:13 smithi022 ceph-mon[24777]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:37:13.890 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:13 smithi022 ceph-mon[24777]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:37:13.890 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:13 smithi022 ceph-mon[24777]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:37:13.890 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:13 smithi022 ceph-mon[24777]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:37:13.890 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:13 smithi022 ceph-mon[24777]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:37:13.891 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:13 smithi022 ceph-mon[24777]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:37:13.891 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:13 smithi022 ceph-mon[24777]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:37:13.891 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:13 smithi022 ceph-mon[24777]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:37:13.891 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:13 smithi022 ceph-mon[24777]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:37:13.891 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:13 smithi022 ceph-mon[24777]: Reconfiguring alertmanager.smithi022 (dependencies changed)... 2024-03-21T12:37:13.891 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:13 smithi022 ceph-mon[24777]: Reconfiguring daemon alertmanager.smithi022 on smithi022 2024-03-21T12:37:14.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:13 smithi189 ceph-mon[27080]: Updating smithi189:/etc/ceph/ceph.client.admin.keyring 2024-03-21T12:37:14.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:13 smithi189 ceph-mon[27080]: Updating smithi022:/etc/ceph/ceph.client.admin.keyring 2024-03-21T12:37:14.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:13 smithi189 ceph-mon[27080]: Updating smithi189:/var/lib/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/config/ceph.client.admin.keyring 2024-03-21T12:37:14.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:13 smithi189 ceph-mon[27080]: Updating smithi022:/var/lib/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/config/ceph.client.admin.keyring 2024-03-21T12:37:14.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:13 smithi189 ceph-mon[27080]: pgmap v13: 97 pgs: 97 active+clean; 334 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.7 MiB/s wr, 628 op/s 2024-03-21T12:37:14.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:13 smithi189 ceph-mon[27080]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:37:14.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:13 smithi189 ceph-mon[27080]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:37:14.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:13 smithi189 ceph-mon[27080]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:37:14.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:13 smithi189 ceph-mon[27080]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:37:14.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:13 smithi189 ceph-mon[27080]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:37:14.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:13 smithi189 ceph-mon[27080]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:37:14.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:13 smithi189 ceph-mon[27080]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:37:14.037 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:13 smithi189 ceph-mon[27080]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:37:14.037 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:13 smithi189 ceph-mon[27080]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:37:14.037 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:13 smithi189 ceph-mon[27080]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:37:14.037 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:13 smithi189 ceph-mon[27080]: Reconfiguring alertmanager.smithi022 (dependencies changed)... 2024-03-21T12:37:14.037 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:13 smithi189 ceph-mon[27080]: Reconfiguring daemon alertmanager.smithi022 on smithi022 2024-03-21T12:37:14.829 INFO:teuthology.orchestra.run.smithi022.stdout: 5 81099 14.03 MB/sec execute 14 sec latency 24.530 ms 2024-03-21T12:37:15.829 INFO:teuthology.orchestra.run.smithi022.stdout: 5 81650 14.39 MB/sec execute 15 sec latency 29.626 ms 2024-03-21T12:37:15.907 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:15 smithi022 ceph-mon[24777]: pgmap v14: 97 pgs: 97 active+clean; 326 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 9.2 MiB/s wr, 899 op/s 2024-03-21T12:37:16.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:15 smithi189 ceph-mon[27080]: pgmap v14: 97 pgs: 97 active+clean; 326 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 9.2 MiB/s wr, 899 op/s 2024-03-21T12:37:16.830 INFO:teuthology.orchestra.run.smithi022.stdout: 5 82117 14.50 MB/sec execute 16 sec latency 27.766 ms 2024-03-21T12:37:17.829 INFO:teuthology.orchestra.run.smithi022.stdout: 5 82428 14.10 MB/sec execute 17 sec latency 70.578 ms 2024-03-21T12:37:17.933 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:17 smithi022 ceph-mon[24777]: pgmap v15: 97 pgs: 97 active+clean; 326 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 5.7 MiB/s wr, 590 op/s 2024-03-21T12:37:18.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:17 smithi189 ceph-mon[27080]: pgmap v15: 97 pgs: 97 active+clean; 326 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 5.7 MiB/s wr, 590 op/s 2024-03-21T12:37:18.829 INFO:teuthology.orchestra.run.smithi022.stdout: 5 82967 13.69 MB/sec execute 18 sec latency 39.707 ms 2024-03-21T12:37:19.829 INFO:teuthology.orchestra.run.smithi022.stdout: 5 83619 14.11 MB/sec execute 19 sec latency 19.216 ms 2024-03-21T12:37:19.947 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:19 smithi022 ceph-mon[24777]: pgmap v16: 97 pgs: 97 active+clean; 326 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 5.7 MiB/s wr, 590 op/s 2024-03-21T12:37:20.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:19 smithi189 ceph-mon[27080]: pgmap v16: 97 pgs: 97 active+clean; 326 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 5.7 MiB/s wr, 590 op/s 2024-03-21T12:37:20.829 INFO:teuthology.orchestra.run.smithi022.stdout: 5 84356 14.16 MB/sec execute 20 sec latency 37.792 ms 2024-03-21T12:37:21.711 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:21 smithi022 ceph-mon[24777]: pgmap v17: 97 pgs: 97 active+clean; 327 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 8.9 MiB/s wr, 882 op/s 2024-03-21T12:37:21.829 INFO:teuthology.orchestra.run.smithi022.stdout: 5 84882 14.39 MB/sec execute 21 sec latency 43.133 ms 2024-03-21T12:37:22.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:21 smithi189 ceph-mon[27080]: pgmap v17: 97 pgs: 97 active+clean; 327 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 8.9 MiB/s wr, 882 op/s 2024-03-21T12:37:22.829 INFO:teuthology.orchestra.run.smithi022.stdout: 5 85523 14.65 MB/sec execute 22 sec latency 33.960 ms 2024-03-21T12:37:23.242 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:23 smithi022 ceph-mon[24777]: pgmap v18: 97 pgs: 97 active+clean; 327 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.7 MiB/s wr, 562 op/s 2024-03-21T12:37:23.242 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:23 smithi022 ceph-mon[24777]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:37:23.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:23 smithi189 ceph-mon[27080]: pgmap v18: 97 pgs: 97 active+clean; 327 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.7 MiB/s wr, 562 op/s 2024-03-21T12:37:23.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:23 smithi189 ceph-mon[27080]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:37:23.829 INFO:teuthology.orchestra.run.smithi022.stdout: 5 85946 14.55 MB/sec execute 23 sec latency 33.839 ms 2024-03-21T12:37:24.830 INFO:teuthology.orchestra.run.smithi022.stdout: 5 86518 14.25 MB/sec execute 24 sec latency 58.891 ms 2024-03-21T12:37:25.511 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:25 smithi022 ceph-mon[24777]: pgmap v19: 97 pgs: 97 active+clean; 331 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.6 MiB/s wr, 924 op/s 2024-03-21T12:37:25.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:25 smithi189 ceph-mon[27080]: pgmap v19: 97 pgs: 97 active+clean; 331 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.6 MiB/s wr, 924 op/s 2024-03-21T12:37:25.829 INFO:teuthology.orchestra.run.smithi022.stdout: 5 87093 14.32 MB/sec execute 25 sec latency 93.394 ms 2024-03-21T12:37:26.829 INFO:teuthology.orchestra.run.smithi022.stdout: 5 87867 14.55 MB/sec execute 26 sec latency 26.823 ms 2024-03-21T12:37:27.529 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:27 smithi022 ceph-mon[24777]: pgmap v20: 97 pgs: 97 active+clean; 331 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 8.0 KiB/s rd, 7.1 MiB/s wr, 653 op/s 2024-03-21T12:37:27.788 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:27 smithi189 ceph-mon[27080]: pgmap v20: 97 pgs: 97 active+clean; 331 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 8.0 KiB/s rd, 7.1 MiB/s wr, 653 op/s 2024-03-21T12:37:27.829 INFO:teuthology.orchestra.run.smithi022.stdout: 5 88298 14.55 MB/sec execute 27 sec latency 35.882 ms 2024-03-21T12:37:28.829 INFO:teuthology.orchestra.run.smithi022.stdout: 5 88862 14.58 MB/sec execute 28 sec latency 27.492 ms 2024-03-21T12:37:29.476 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:29 smithi022 ceph-mon[24777]: pgmap v21: 97 pgs: 97 active+clean; 331 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 8.0 KiB/s rd, 7.1 MiB/s wr, 653 op/s 2024-03-21T12:37:29.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:29 smithi189 ceph-mon[27080]: pgmap v21: 97 pgs: 97 active+clean; 331 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 8.0 KiB/s rd, 7.1 MiB/s wr, 653 op/s 2024-03-21T12:37:29.830 INFO:teuthology.orchestra.run.smithi022.stdout: 5 89275 14.61 MB/sec execute 29 sec latency 41.859 ms 2024-03-21T12:37:30.829 INFO:teuthology.orchestra.run.smithi022.stdout: 5 89581 14.37 MB/sec execute 30 sec latency 41.353 ms 2024-03-21T12:37:31.192 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:31 smithi022 ceph-mon[24777]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:37:31.193 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:31 smithi022 ceph-mon[24777]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:37:31.193 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:31 smithi022 ceph-mon[24777]: Reconfiguring grafana.smithi022 (dependencies changed)... 2024-03-21T12:37:31.193 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:31 smithi022 ceph-mon[24777]: Regenerating cephadm self-signed grafana TLS certificates 2024-03-21T12:37:31.193 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:31 smithi022 ceph-mon[24777]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:37:31.193 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:31 smithi022 ceph-mon[24777]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:37:31.193 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:31 smithi022 ceph-mon[24777]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-03-21T12:37:31.193 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:31 smithi022 ceph-mon[24777]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-03-21T12:37:31.193 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:31 smithi022 ceph-mon[24777]: Reconfiguring daemon grafana.smithi022 on smithi022 2024-03-21T12:37:31.193 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:31 smithi022 ceph-mon[24777]: pgmap v22: 97 pgs: 97 active+clean; 331 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 10 MiB/s wr, 982 op/s 2024-03-21T12:37:31.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:31 smithi189 ceph-mon[27080]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:37:31.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:31 smithi189 ceph-mon[27080]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:37:31.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:31 smithi189 ceph-mon[27080]: Reconfiguring grafana.smithi022 (dependencies changed)... 2024-03-21T12:37:31.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:31 smithi189 ceph-mon[27080]: Regenerating cephadm self-signed grafana TLS certificates 2024-03-21T12:37:31.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:31 smithi189 ceph-mon[27080]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:37:31.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:31 smithi189 ceph-mon[27080]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:37:31.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:31 smithi189 ceph-mon[27080]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-03-21T12:37:31.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:31 smithi189 ceph-mon[27080]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-03-21T12:37:31.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:31 smithi189 ceph-mon[27080]: Reconfiguring daemon grafana.smithi022 on smithi022 2024-03-21T12:37:31.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:31 smithi189 ceph-mon[27080]: pgmap v22: 97 pgs: 97 active+clean; 331 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 10 MiB/s wr, 982 op/s 2024-03-21T12:37:31.830 INFO:teuthology.orchestra.run.smithi022.stdout: 5 90221 14.18 MB/sec execute 31 sec latency 27.500 ms 2024-03-21T12:37:32.829 INFO:teuthology.orchestra.run.smithi022.stdout: 5 90864 14.52 MB/sec execute 32 sec latency 17.498 ms 2024-03-21T12:37:33.469 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:33 smithi022 ceph-mon[24777]: pgmap v23: 97 pgs: 97 active+clean; 331 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 9.6 KiB/s rd, 7.3 MiB/s wr, 691 op/s 2024-03-21T12:37:33.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:33 smithi189 ceph-mon[27080]: pgmap v23: 97 pgs: 97 active+clean; 331 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 9.6 KiB/s rd, 7.3 MiB/s wr, 691 op/s 2024-03-21T12:37:33.829 INFO:teuthology.orchestra.run.smithi022.stdout: 5 91439 14.37 MB/sec execute 33 sec latency 46.680 ms 2024-03-21T12:37:34.832 INFO:teuthology.orchestra.run.smithi022.stdout: 5 91795 14.26 MB/sec execute 34 sec latency 31.950 ms 2024-03-21T12:37:35.507 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:35 smithi022 ceph-mon[24777]: pgmap v24: 97 pgs: 97 active+clean; 326 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 10 MiB/s wr, 995 op/s 2024-03-21T12:37:35.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:35 smithi189 ceph-mon[27080]: pgmap v24: 97 pgs: 97 active+clean; 326 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 10 MiB/s wr, 995 op/s 2024-03-21T12:37:35.832 INFO:teuthology.orchestra.run.smithi022.stdout: 5 92564 14.59 MB/sec execute 35 sec latency 40.840 ms 2024-03-21T12:37:36.832 INFO:teuthology.orchestra.run.smithi022.stdout: 5 93300 14.73 MB/sec execute 36 sec latency 18.805 ms 2024-03-21T12:37:37.521 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:37 smithi022 ceph-mon[24777]: pgmap v25: 97 pgs: 97 active+clean; 326 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 6.2 MiB/s wr, 633 op/s 2024-03-21T12:37:37.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:37 smithi189 ceph-mon[27080]: pgmap v25: 97 pgs: 97 active+clean; 326 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 6.2 MiB/s wr, 633 op/s 2024-03-21T12:37:37.832 INFO:teuthology.orchestra.run.smithi022.stdout: 5 93816 14.50 MB/sec execute 37 sec latency 44.235 ms 2024-03-21T12:37:38.537 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:38 smithi022 ceph-mon[24777]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:37:38.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:38 smithi189 ceph-mon[27080]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:37:38.832 INFO:teuthology.orchestra.run.smithi022.stdout: 5 94343 14.73 MB/sec execute 38 sec latency 28.656 ms 2024-03-21T12:37:39.546 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:39 smithi022 ceph-mon[24777]: pgmap v26: 97 pgs: 97 active+clean; 326 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 6.2 MiB/s wr, 633 op/s 2024-03-21T12:37:39.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:39 smithi189 ceph-mon[27080]: pgmap v26: 97 pgs: 97 active+clean; 326 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 6.2 MiB/s wr, 633 op/s 2024-03-21T12:37:39.832 INFO:teuthology.orchestra.run.smithi022.stdout: 5 94892 14.58 MB/sec execute 39 sec latency 39.679 ms 2024-03-21T12:37:40.832 INFO:teuthology.orchestra.run.smithi022.stdout: 5 95347 14.51 MB/sec execute 40 sec latency 20.468 ms 2024-03-21T12:37:41.568 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:41 smithi022 ceph-mon[24777]: pgmap v27: 97 pgs: 97 active+clean; 335 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 10 MiB/s wr, 981 op/s 2024-03-21T12:37:41.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:41 smithi189 ceph-mon[27080]: pgmap v27: 97 pgs: 97 active+clean; 335 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 10 MiB/s wr, 981 op/s 2024-03-21T12:37:41.832 INFO:teuthology.orchestra.run.smithi022.stdout: 5 96084 14.78 MB/sec execute 41 sec latency 61.445 ms 2024-03-21T12:37:42.566 INFO:teuthology.orchestra.run.smithi022.stdout:true 2024-03-21T12:37:42.832 INFO:teuthology.orchestra.run.smithi022.stdout: 5 96595 14.80 MB/sec execute 42 sec latency 31.034 ms 2024-03-21T12:37:43.351 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:43 smithi022 ceph-mon[24777]: pgmap v28: 97 pgs: 97 active+clean; 335 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.3 KiB/s rd, 6.7 MiB/s wr, 652 op/s 2024-03-21T12:37:43.352 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:43 smithi022 ceph-mon[24777]: from='client.14720 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:37:43.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:43 smithi189 ceph-mon[27080]: pgmap v28: 97 pgs: 97 active+clean; 335 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.3 KiB/s rd, 6.7 MiB/s wr, 652 op/s 2024-03-21T12:37:43.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:43 smithi189 ceph-mon[27080]: from='client.14720 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:37:43.713 INFO:teuthology.orchestra.run.smithi022.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-21T12:37:43.713 INFO:teuthology.orchestra.run.smithi022.stdout:alertmanager.smithi022 smithi022 *:9093,9094 starting - - - - 2024-03-21T12:37:43.713 INFO:teuthology.orchestra.run.smithi022.stdout:crash.smithi022 smithi022 running (9m) 35s ago 9m 6405k - 17.2.0 e1d6a67b021e 47deb70fd250 2024-03-21T12:37:43.713 INFO:teuthology.orchestra.run.smithi022.stdout:crash.smithi189 smithi189 running (8m) 37s ago 8m 6405k - 17.2.0 e1d6a67b021e 9a2e0e9b1809 2024-03-21T12:37:43.713 INFO:teuthology.orchestra.run.smithi022.stdout:grafana.smithi022 smithi022 *:3000 running (7m) 35s ago 8m 55.2M - 8.3.5 dad864ee21e9 3c9ff131fc6b 2024-03-21T12:37:43.714 INFO:teuthology.orchestra.run.smithi022.stdout:mds.foofs.smithi022.bcnxho smithi022 running (3m) 35s ago 3m 36.7M - 17.2.0 e1d6a67b021e db26b67b8ee7 2024-03-21T12:37:43.714 INFO:teuthology.orchestra.run.smithi022.stdout:mds.foofs.smithi189.lluvko smithi189 running (3m) 37s ago 3m 15.3M - 17.2.0 e1d6a67b021e cf153aff1387 2024-03-21T12:37:43.714 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.smithi022.fektzs smithi022 *:9283 running (10m) 35s ago 10m 346M - 17.2.0 e1d6a67b021e ca06c9c2fb41 2024-03-21T12:37:43.714 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.smithi189.cnxzvu smithi189 *:8443,9283 running (61s) 37s ago 8m 488M - 19.0.0-2267-g960fbbe4 19e12293512f 07f3cf369ff7 2024-03-21T12:37:43.714 INFO:teuthology.orchestra.run.smithi022.stdout:mon.smithi022 smithi022 running (10m) 35s ago 10m 49.9M 2048M 17.2.0 e1d6a67b021e 0d248775e0e5 2024-03-21T12:37:43.714 INFO:teuthology.orchestra.run.smithi022.stdout:mon.smithi189 smithi189 running (8m) 37s ago 8m 40.1M 2048M 17.2.0 e1d6a67b021e 82da5f3cd49a 2024-03-21T12:37:43.714 INFO:teuthology.orchestra.run.smithi022.stdout:nfs.foo.0.0.smithi022.ynhbbq smithi022 *:2049 running (2m) 35s ago 2m 250M - 4.0 e1d6a67b021e 33e48e64c60b 2024-03-21T12:37:43.714 INFO:teuthology.orchestra.run.smithi022.stdout:nfs.foo.1.0.smithi189.yzsqml smithi189 *:2049 running (2m) 37s ago 2m 73.3M - 4.0 e1d6a67b021e 48fab2cbdde2 2024-03-21T12:37:43.714 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.smithi022 smithi022 *:9100 running (8m) 35s ago 8m 17.5M - 1.3.1 1dbe0e931976 cea183555968 2024-03-21T12:37:43.714 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.smithi189 smithi189 *:9100 running (8m) 37s ago 8m 22.6M - 1.3.1 1dbe0e931976 5db95224aff9 2024-03-21T12:37:43.714 INFO:teuthology.orchestra.run.smithi022.stdout:osd.0 smithi022 running (7m) 35s ago 7m 294M 1937M 17.2.0 e1d6a67b021e 54e6631bd10f 2024-03-21T12:37:43.714 INFO:teuthology.orchestra.run.smithi022.stdout:osd.1 smithi022 running (7m) 35s ago 7m 233M 1937M 17.2.0 e1d6a67b021e 68165c5d09cb 2024-03-21T12:37:43.714 INFO:teuthology.orchestra.run.smithi022.stdout:osd.2 smithi022 running (6m) 35s ago 6m 184M 1937M 17.2.0 e1d6a67b021e fbec9bef8433 2024-03-21T12:37:43.714 INFO:teuthology.orchestra.run.smithi022.stdout:osd.3 smithi022 running (6m) 35s ago 6m 256M 1937M 17.2.0 e1d6a67b021e b1462ffc1b9e 2024-03-21T12:37:43.715 INFO:teuthology.orchestra.run.smithi022.stdout:osd.4 smithi189 running (5m) 37s ago 5m 276M 2704M 17.2.0 e1d6a67b021e 1bedc4eb02ed 2024-03-21T12:37:43.715 INFO:teuthology.orchestra.run.smithi022.stdout:osd.5 smithi189 running (5m) 37s ago 5m 313M 2704M 17.2.0 e1d6a67b021e 25fbab9786a5 2024-03-21T12:37:43.715 INFO:teuthology.orchestra.run.smithi022.stdout:osd.6 smithi189 running (5m) 37s ago 5m 262M 2704M 17.2.0 e1d6a67b021e 0c192fcfd65b 2024-03-21T12:37:43.715 INFO:teuthology.orchestra.run.smithi022.stdout:osd.7 smithi189 running (4m) 37s ago 4m 384M 2704M 17.2.0 e1d6a67b021e 3d0723e1f863 2024-03-21T12:37:43.715 INFO:teuthology.orchestra.run.smithi022.stdout:prometheus.smithi022 smithi022 *:9095 running (7m) 35s ago 7m 64.2M - 2.33.4 514e6a882f6e 20a0eef27ec8 2024-03-21T12:37:43.832 INFO:teuthology.orchestra.run.smithi022.stdout: 5 97097 14.61 MB/sec execute 43 sec latency 49.873 ms 2024-03-21T12:37:44.355 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:44 smithi022 ceph-mon[24777]: from='client.14724 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:37:44.356 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:44 smithi022 ceph-mon[24777]: from='client.24515 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:37:44.398 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-03-21T12:37:44.399 INFO:teuthology.orchestra.run.smithi022.stdout: "mon": { 2024-03-21T12:37:44.399 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-21T12:37:44.399 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:37:44.399 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr": { 2024-03-21T12:37:44.399 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-03-21T12:37:44.399 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 1 2024-03-21T12:37:44.399 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:37:44.399 INFO:teuthology.orchestra.run.smithi022.stdout: "osd": { 2024-03-21T12:37:44.399 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-03-21T12:37:44.399 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:37:44.399 INFO:teuthology.orchestra.run.smithi022.stdout: "mds": { 2024-03-21T12:37:44.399 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-21T12:37:44.399 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:37:44.400 INFO:teuthology.orchestra.run.smithi022.stdout: "overall": { 2024-03-21T12:37:44.400 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-03-21T12:37:44.400 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 1 2024-03-21T12:37:44.400 INFO:teuthology.orchestra.run.smithi022.stdout: } 2024-03-21T12:37:44.400 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-03-21T12:37:44.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:44 smithi189 ceph-mon[27080]: from='client.14724 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:37:44.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:44 smithi189 ceph-mon[27080]: from='client.24515 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:37:44.832 INFO:teuthology.orchestra.run.smithi022.stdout: 5 97496 14.50 MB/sec execute 44 sec latency 50.222 ms 2024-03-21T12:37:44.948 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-03-21T12:37:44.948 INFO:teuthology.orchestra.run.smithi022.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93", 2024-03-21T12:37:44.948 INFO:teuthology.orchestra.run.smithi022.stdout: "in_progress": true, 2024-03-21T12:37:44.948 INFO:teuthology.orchestra.run.smithi022.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-21T12:37:44.948 INFO:teuthology.orchestra.run.smithi022.stdout: "services_complete": [], 2024-03-21T12:37:44.948 INFO:teuthology.orchestra.run.smithi022.stdout: "progress": "1/23 daemons upgraded", 2024-03-21T12:37:44.948 INFO:teuthology.orchestra.run.smithi022.stdout: "message": "", 2024-03-21T12:37:44.948 INFO:teuthology.orchestra.run.smithi022.stdout: "is_paused": false 2024-03-21T12:37:44.948 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-03-21T12:37:45.368 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:45 smithi022 ceph-mon[24777]: from='client.? 172.21.15.22:0/1418482882' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:37:45.368 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:45 smithi022 ceph-mon[24777]: pgmap v29: 97 pgs: 97 active+clean; 331 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 10 MiB/s wr, 967 op/s 2024-03-21T12:37:45.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:45 smithi189 ceph-mon[27080]: from='client.? 172.21.15.22:0/1418482882' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:37:45.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:45 smithi189 ceph-mon[27080]: pgmap v29: 97 pgs: 97 active+clean; 331 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 10 MiB/s wr, 967 op/s 2024-03-21T12:37:45.615 INFO:teuthology.orchestra.run.smithi022.stdout:HEALTH_OK 2024-03-21T12:37:45.835 INFO:teuthology.orchestra.run.smithi022.stdout: 5 97977 14.58 MB/sec execute 45 sec latency 48.387 ms 2024-03-21T12:37:46.387 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:46 smithi022 ceph-mon[24777]: from='client.14736 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:37:46.388 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:46 smithi022 ceph-mon[24777]: from='client.? 172.21.15.22:0/733697607' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-21T12:37:46.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:46 smithi189 ceph-mon[27080]: from='client.14736 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:37:46.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:46 smithi189 ceph-mon[27080]: from='client.? 172.21.15.22:0/733697607' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-21T12:37:46.837 INFO:teuthology.orchestra.run.smithi022.stdout: 5 98459 14.45 MB/sec execute 46 sec latency 45.590 ms 2024-03-21T12:37:47.394 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:47 smithi022 ceph-mon[24777]: pgmap v30: 97 pgs: 97 active+clean; 331 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.9 KiB/s rd, 7.1 MiB/s wr, 663 op/s 2024-03-21T12:37:47.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:47 smithi189 ceph-mon[27080]: pgmap v30: 97 pgs: 97 active+clean; 331 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.9 KiB/s rd, 7.1 MiB/s wr, 663 op/s 2024-03-21T12:37:47.840 INFO:teuthology.orchestra.run.smithi022.stdout: 5 98810 14.33 MB/sec execute 47 sec latency 49.191 ms 2024-03-21T12:37:48.841 INFO:teuthology.orchestra.run.smithi022.stdout: 5 99280 14.40 MB/sec execute 48 sec latency 29.101 ms 2024-03-21T12:37:49.681 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:49 smithi022 ceph-mon[24777]: pgmap v31: 97 pgs: 97 active+clean; 331 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.9 KiB/s rd, 7.1 MiB/s wr, 663 op/s 2024-03-21T12:37:49.681 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:49 smithi022 ceph-mon[24777]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:37:49.682 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:49 smithi022 ceph-mon[24777]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:37:49.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:49 smithi189 ceph-mon[27080]: pgmap v31: 97 pgs: 97 active+clean; 331 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.9 KiB/s rd, 7.1 MiB/s wr, 663 op/s 2024-03-21T12:37:49.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:49 smithi189 ceph-mon[27080]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:37:49.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:49 smithi189 ceph-mon[27080]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:37:49.842 INFO:teuthology.orchestra.run.smithi022.stdout: 5 99770 14.45 MB/sec execute 49 sec latency 40.778 ms 2024-03-21T12:37:50.696 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:50 smithi022 ceph-mon[24777]: Reconfiguring node-exporter.smithi022 (dependencies changed)... 2024-03-21T12:37:50.697 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:50 smithi022 ceph-mon[24777]: Reconfiguring daemon node-exporter.smithi022 on smithi022 2024-03-21T12:37:50.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:50 smithi189 ceph-mon[27080]: Reconfiguring node-exporter.smithi022 (dependencies changed)... 2024-03-21T12:37:50.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:50 smithi189 ceph-mon[27080]: Reconfiguring daemon node-exporter.smithi022 on smithi022 2024-03-21T12:37:50.845 INFO:teuthology.orchestra.run.smithi022.stdout: 5 100093 14.34 MB/sec execute 50 sec latency 31.945 ms 2024-03-21T12:37:51.564 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:51 smithi022 ceph-mon[24777]: pgmap v32: 97 pgs: 97 active+clean; 327 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.7 MiB/s wr, 926 op/s 2024-03-21T12:37:51.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:51 smithi189 ceph-mon[27080]: pgmap v32: 97 pgs: 97 active+clean; 327 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.7 MiB/s wr, 926 op/s 2024-03-21T12:37:51.842 INFO:teuthology.orchestra.run.smithi022.stdout: 5 100442 14.15 MB/sec execute 51 sec latency 44.998 ms 2024-03-21T12:37:52.842 INFO:teuthology.orchestra.run.smithi022.stdout: 5 100803 13.95 MB/sec execute 52 sec latency 34.998 ms 2024-03-21T12:37:53.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:53 smithi022 ceph-mon[24777]: pgmap v33: 97 pgs: 97 active+clean; 327 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.9 MiB/s wr, 578 op/s 2024-03-21T12:37:53.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:53 smithi022 ceph-mon[24777]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:37:53.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:53 smithi189 ceph-mon[27080]: pgmap v33: 97 pgs: 97 active+clean; 327 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.9 MiB/s wr, 578 op/s 2024-03-21T12:37:53.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:53 smithi189 ceph-mon[27080]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:37:53.843 INFO:teuthology.orchestra.run.smithi022.stdout: 5 101096 13.89 MB/sec execute 53 sec latency 43.114 ms 2024-03-21T12:37:54.842 INFO:teuthology.orchestra.run.smithi022.stdout: 5 101533 13.94 MB/sec execute 54 sec latency 54.284 ms 2024-03-21T12:37:54.862 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:54 smithi022 ceph-mon[24777]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:37:54.862 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:54 smithi022 ceph-mon[24777]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:37:54.862 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:54 smithi022 ceph-mon[24777]: Reconfiguring prometheus.smithi022 (dependencies changed)... 2024-03-21T12:37:54.862 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:54 smithi022 ceph-mon[24777]: Reconfiguring daemon prometheus.smithi022 on smithi022 2024-03-21T12:37:55.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:54 smithi189 ceph-mon[27080]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:37:55.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:54 smithi189 ceph-mon[27080]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:37:55.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:54 smithi189 ceph-mon[27080]: Reconfiguring prometheus.smithi022 (dependencies changed)... 2024-03-21T12:37:55.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:54 smithi189 ceph-mon[27080]: Reconfiguring daemon prometheus.smithi022 on smithi022 2024-03-21T12:37:55.842 INFO:teuthology.orchestra.run.smithi022.stdout: 5 102056 13.83 MB/sec execute 55 sec latency 46.056 ms 2024-03-21T12:37:55.867 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:55 smithi022 ceph-mon[24777]: pgmap v34: 97 pgs: 97 active+clean; 329 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.3 MiB/s wr, 815 op/s 2024-03-21T12:37:56.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:55 smithi189 ceph-mon[27080]: pgmap v34: 97 pgs: 97 active+clean; 329 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.3 MiB/s wr, 815 op/s 2024-03-21T12:37:56.843 INFO:teuthology.orchestra.run.smithi022.stdout: 5 102390 13.76 MB/sec execute 56 sec latency 30.672 ms 2024-03-21T12:37:57.842 INFO:teuthology.orchestra.run.smithi022.stdout: 5 102804 13.80 MB/sec execute 57 sec latency 48.755 ms 2024-03-21T12:37:57.896 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:57 smithi022 ceph-mon[24777]: pgmap v35: 97 pgs: 97 active+clean; 329 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.0 MiB/s wr, 500 op/s 2024-03-21T12:37:58.037 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:57 smithi189 ceph-mon[27080]: pgmap v35: 97 pgs: 97 active+clean; 329 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.0 MiB/s wr, 500 op/s 2024-03-21T12:37:58.843 INFO:teuthology.orchestra.run.smithi022.stdout: 5 103293 13.86 MB/sec execute 58 sec latency 59.771 ms 2024-03-21T12:37:59.720 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:37:59 smithi022 ceph-mon[24777]: pgmap v36: 97 pgs: 97 active+clean; 329 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.0 MiB/s wr, 500 op/s 2024-03-21T12:37:59.843 INFO:teuthology.orchestra.run.smithi022.stdout: 5 103587 13.77 MB/sec execute 59 sec latency 56.398 ms 2024-03-21T12:38:00.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:37:59 smithi189 ceph-mon[27080]: pgmap v36: 97 pgs: 97 active+clean; 329 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.0 MiB/s wr, 500 op/s 2024-03-21T12:38:00.842 INFO:teuthology.orchestra.run.smithi022.stdout: 5 104035 13.63 MB/sec execute 60 sec latency 38.734 ms 2024-03-21T12:38:01.741 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:01 smithi022 ceph-mon[24777]: pgmap v37: 97 pgs: 97 active+clean; 335 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.8 MiB/s wr, 748 op/s 2024-03-21T12:38:01.842 INFO:teuthology.orchestra.run.smithi022.stdout: 5 104432 13.49 MB/sec execute 61 sec latency 25.855 ms 2024-03-21T12:38:02.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:01 smithi189 ceph-mon[27080]: pgmap v37: 97 pgs: 97 active+clean; 335 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.8 MiB/s wr, 748 op/s 2024-03-21T12:38:02.843 INFO:teuthology.orchestra.run.smithi022.stdout: 5 104827 13.49 MB/sec execute 62 sec latency 32.248 ms 2024-03-21T12:38:03.275 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:03 smithi022 ceph-mon[24777]: pgmap v38: 97 pgs: 97 active+clean; 335 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.2 MiB/s wr, 485 op/s 2024-03-21T12:38:03.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:03 smithi189 ceph-mon[27080]: pgmap v38: 97 pgs: 97 active+clean; 335 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.2 MiB/s wr, 485 op/s 2024-03-21T12:38:03.843 INFO:teuthology.orchestra.run.smithi022.stdout: 5 105271 13.53 MB/sec execute 63 sec latency 58.943 ms 2024-03-21T12:38:04.843 INFO:teuthology.orchestra.run.smithi022.stdout: 5 105799 13.46 MB/sec execute 64 sec latency 29.168 ms 2024-03-21T12:38:05.556 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:05 smithi022 ceph-mon[24777]: pgmap v39: 97 pgs: 97 active+clean; 327 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.6 MiB/s wr, 732 op/s 2024-03-21T12:38:05.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:05 smithi189 ceph-mon[27080]: pgmap v39: 97 pgs: 97 active+clean; 327 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.6 MiB/s wr, 732 op/s 2024-03-21T12:38:05.843 INFO:teuthology.orchestra.run.smithi022.stdout: 5 106064 13.36 MB/sec execute 65 sec latency 38.760 ms 2024-03-21T12:38:06.843 INFO:teuthology.orchestra.run.smithi022.stdout: 5 106515 13.40 MB/sec execute 66 sec latency 43.773 ms 2024-03-21T12:38:07.600 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:07 smithi022 ceph-mon[24777]: pgmap v40: 97 pgs: 97 active+clean; 327 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.2 MiB/s wr, 495 op/s 2024-03-21T12:38:07.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:07 smithi189 ceph-mon[27080]: pgmap v40: 97 pgs: 97 active+clean; 327 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.2 MiB/s wr, 495 op/s 2024-03-21T12:38:07.843 INFO:teuthology.orchestra.run.smithi022.stdout: 5 106979 13.49 MB/sec execute 67 sec latency 60.337 ms 2024-03-21T12:38:08.604 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:08 smithi022 ceph-mon[24777]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:38:08.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:08 smithi189 ceph-mon[27080]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:38:08.843 INFO:teuthology.orchestra.run.smithi022.stdout: 5 107467 13.42 MB/sec execute 68 sec latency 45.173 ms 2024-03-21T12:38:09.608 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:09 smithi022 ceph-mon[24777]: pgmap v41: 97 pgs: 97 active+clean; 327 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.2 MiB/s wr, 495 op/s 2024-03-21T12:38:09.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:09 smithi189 ceph-mon[27080]: pgmap v41: 97 pgs: 97 active+clean; 327 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.2 MiB/s wr, 495 op/s 2024-03-21T12:38:09.843 INFO:teuthology.orchestra.run.smithi022.stdout: 5 107986 13.32 MB/sec execute 69 sec latency 81.325 ms 2024-03-21T12:38:10.843 INFO:teuthology.orchestra.run.smithi022.stdout: 5 108614 13.49 MB/sec execute 70 sec latency 44.550 ms 2024-03-21T12:38:11.637 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:11 smithi022 ceph-mon[24777]: pgmap v42: 97 pgs: 97 active+clean; 332 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 7.9 MiB/s wr, 753 op/s 2024-03-21T12:38:11.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:11 smithi189 ceph-mon[27080]: pgmap v42: 97 pgs: 97 active+clean; 332 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 7.9 MiB/s wr, 753 op/s 2024-03-21T12:38:11.843 INFO:teuthology.orchestra.run.smithi022.stdout: 5 109348 13.50 MB/sec execute 71 sec latency 33.927 ms 2024-03-21T12:38:12.843 INFO:teuthology.orchestra.run.smithi022.stdout: 5 109856 13.55 MB/sec execute 72 sec latency 26.975 ms 2024-03-21T12:38:13.416 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:13 smithi022 ceph-mon[24777]: pgmap v43: 97 pgs: 97 active+clean; 332 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.2 MiB/s wr, 506 op/s 2024-03-21T12:38:13.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:13 smithi189 ceph-mon[27080]: pgmap v43: 97 pgs: 97 active+clean; 332 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.2 MiB/s wr, 506 op/s 2024-03-21T12:38:13.843 INFO:teuthology.orchestra.run.smithi022.stdout: 5 110384 13.59 MB/sec execute 73 sec latency 28.536 ms 2024-03-21T12:38:14.843 INFO:teuthology.orchestra.run.smithi022.stdout: 5 110704 13.56 MB/sec execute 74 sec latency 37.396 ms 2024-03-21T12:38:15.728 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:15 smithi022 ceph-mon[24777]: pgmap v44: 97 pgs: 97 active+clean; 337 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.8 MiB/s wr, 851 op/s 2024-03-21T12:38:15.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:15 smithi189 ceph-mon[27080]: pgmap v44: 97 pgs: 97 active+clean; 337 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.8 MiB/s wr, 851 op/s 2024-03-21T12:38:15.843 INFO:teuthology.orchestra.run.smithi022.stdout: 5 111030 13.45 MB/sec execute 75 sec latency 49.175 ms 2024-03-21T12:38:16.097 INFO:teuthology.orchestra.run.smithi022.stdout:true 2024-03-21T12:38:16.843 INFO:teuthology.orchestra.run.smithi022.stdout: 5 111407 13.32 MB/sec execute 76 sec latency 66.462 ms 2024-03-21T12:38:17.048 INFO:teuthology.orchestra.run.smithi022.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-21T12:38:17.048 INFO:teuthology.orchestra.run.smithi022.stdout:alertmanager.smithi022 smithi022 *:9093,9094 starting - - - - 2024-03-21T12:38:17.048 INFO:teuthology.orchestra.run.smithi022.stdout:crash.smithi022 smithi022 running (9m) 68s ago 9m 6405k - 17.2.0 e1d6a67b021e 47deb70fd250 2024-03-21T12:38:17.048 INFO:teuthology.orchestra.run.smithi022.stdout:crash.smithi189 smithi189 running (9m) 71s ago 9m 6405k - 17.2.0 e1d6a67b021e 9a2e0e9b1809 2024-03-21T12:38:17.048 INFO:teuthology.orchestra.run.smithi022.stdout:grafana.smithi022 smithi022 *:3000 starting - - - - 2024-03-21T12:38:17.048 INFO:teuthology.orchestra.run.smithi022.stdout:mds.foofs.smithi022.bcnxho smithi022 running (3m) 68s ago 3m 36.7M - 17.2.0 e1d6a67b021e db26b67b8ee7 2024-03-21T12:38:17.048 INFO:teuthology.orchestra.run.smithi022.stdout:mds.foofs.smithi189.lluvko smithi189 running (3m) 71s ago 3m 15.3M - 17.2.0 e1d6a67b021e cf153aff1387 2024-03-21T12:38:17.048 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.smithi022.fektzs smithi022 *:9283 running (10m) 68s ago 10m 346M - 17.2.0 e1d6a67b021e ca06c9c2fb41 2024-03-21T12:38:17.048 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.smithi189.cnxzvu smithi189 *:8443,9283 running (94s) 71s ago 8m 488M - 19.0.0-2267-g960fbbe4 19e12293512f 07f3cf369ff7 2024-03-21T12:38:17.048 INFO:teuthology.orchestra.run.smithi022.stdout:mon.smithi022 smithi022 running (10m) 68s ago 11m 49.9M 2048M 17.2.0 e1d6a67b021e 0d248775e0e5 2024-03-21T12:38:17.048 INFO:teuthology.orchestra.run.smithi022.stdout:mon.smithi189 smithi189 running (8m) 71s ago 8m 40.1M 2048M 17.2.0 e1d6a67b021e 82da5f3cd49a 2024-03-21T12:38:17.048 INFO:teuthology.orchestra.run.smithi022.stdout:nfs.foo.0.0.smithi022.ynhbbq smithi022 *:2049 running (3m) 68s ago 3m 250M - 4.0 e1d6a67b021e 33e48e64c60b 2024-03-21T12:38:17.049 INFO:teuthology.orchestra.run.smithi022.stdout:nfs.foo.1.0.smithi189.yzsqml smithi189 *:2049 running (3m) 71s ago 3m 73.3M - 4.0 e1d6a67b021e 48fab2cbdde2 2024-03-21T12:38:17.049 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.smithi022 smithi022 *:9100 starting - - - - 2024-03-21T12:38:17.049 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.smithi189 smithi189 *:9100 running (8m) 71s ago 8m 22.6M - 1.3.1 1dbe0e931976 5db95224aff9 2024-03-21T12:38:17.049 INFO:teuthology.orchestra.run.smithi022.stdout:osd.0 smithi022 running (8m) 68s ago 8m 294M 1937M 17.2.0 e1d6a67b021e 54e6631bd10f 2024-03-21T12:38:17.049 INFO:teuthology.orchestra.run.smithi022.stdout:osd.1 smithi022 running (7m) 68s ago 7m 233M 1937M 17.2.0 e1d6a67b021e 68165c5d09cb 2024-03-21T12:38:17.049 INFO:teuthology.orchestra.run.smithi022.stdout:osd.2 smithi022 running (7m) 68s ago 7m 184M 1937M 17.2.0 e1d6a67b021e fbec9bef8433 2024-03-21T12:38:17.049 INFO:teuthology.orchestra.run.smithi022.stdout:osd.3 smithi022 running (6m) 68s ago 6m 256M 1937M 17.2.0 e1d6a67b021e b1462ffc1b9e 2024-03-21T12:38:17.049 INFO:teuthology.orchestra.run.smithi022.stdout:osd.4 smithi189 running (6m) 71s ago 6m 276M 2704M 17.2.0 e1d6a67b021e 1bedc4eb02ed 2024-03-21T12:38:17.049 INFO:teuthology.orchestra.run.smithi022.stdout:osd.5 smithi189 running (6m) 71s ago 6m 313M 2704M 17.2.0 e1d6a67b021e 25fbab9786a5 2024-03-21T12:38:17.049 INFO:teuthology.orchestra.run.smithi022.stdout:osd.6 smithi189 running (5m) 71s ago 5m 262M 2704M 17.2.0 e1d6a67b021e 0c192fcfd65b 2024-03-21T12:38:17.049 INFO:teuthology.orchestra.run.smithi022.stdout:osd.7 smithi189 running (5m) 71s ago 5m 384M 2704M 17.2.0 e1d6a67b021e 3d0723e1f863 2024-03-21T12:38:17.049 INFO:teuthology.orchestra.run.smithi022.stdout:prometheus.smithi022 smithi022 *:9095 running (8m) 68s ago 8m 64.2M - 2.33.4 514e6a882f6e 20a0eef27ec8 2024-03-21T12:38:17.515 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:17 smithi022 ceph-mon[24777]: from='client.14744 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:38:17.516 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:17 smithi022 ceph-mon[24777]: pgmap v45: 97 pgs: 97 active+clean; 337 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.6 KiB/s rd, 6.4 MiB/s wr, 604 op/s 2024-03-21T12:38:17.516 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:17 smithi022 ceph-mon[24777]: from='client.14748 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:38:17.589 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-03-21T12:38:17.590 INFO:teuthology.orchestra.run.smithi022.stdout: "mon": { 2024-03-21T12:38:17.590 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-21T12:38:17.590 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:38:17.590 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr": { 2024-03-21T12:38:17.590 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-03-21T12:38:17.590 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 1 2024-03-21T12:38:17.590 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:38:17.590 INFO:teuthology.orchestra.run.smithi022.stdout: "osd": { 2024-03-21T12:38:17.590 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-03-21T12:38:17.590 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:38:17.590 INFO:teuthology.orchestra.run.smithi022.stdout: "mds": { 2024-03-21T12:38:17.590 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-21T12:38:17.591 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:38:17.591 INFO:teuthology.orchestra.run.smithi022.stdout: "overall": { 2024-03-21T12:38:17.591 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-03-21T12:38:17.591 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 1 2024-03-21T12:38:17.591 INFO:teuthology.orchestra.run.smithi022.stdout: } 2024-03-21T12:38:17.591 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-03-21T12:38:17.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:17 smithi189 ceph-mon[27080]: from='client.14744 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:38:17.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:17 smithi189 ceph-mon[27080]: pgmap v45: 97 pgs: 97 active+clean; 337 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.6 KiB/s rd, 6.4 MiB/s wr, 604 op/s 2024-03-21T12:38:17.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:17 smithi189 ceph-mon[27080]: from='client.14748 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:38:17.843 INFO:teuthology.orchestra.run.smithi022.stdout: 5 111670 13.24 MB/sec execute 77 sec latency 103.215 ms 2024-03-21T12:38:18.060 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-03-21T12:38:18.060 INFO:teuthology.orchestra.run.smithi022.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93", 2024-03-21T12:38:18.060 INFO:teuthology.orchestra.run.smithi022.stdout: "in_progress": true, 2024-03-21T12:38:18.060 INFO:teuthology.orchestra.run.smithi022.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-21T12:38:18.060 INFO:teuthology.orchestra.run.smithi022.stdout: "services_complete": [], 2024-03-21T12:38:18.060 INFO:teuthology.orchestra.run.smithi022.stdout: "progress": "1/23 daemons upgraded", 2024-03-21T12:38:18.060 INFO:teuthology.orchestra.run.smithi022.stdout: "message": "", 2024-03-21T12:38:18.060 INFO:teuthology.orchestra.run.smithi022.stdout: "is_paused": false 2024-03-21T12:38:18.061 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-03-21T12:38:18.578 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:18 smithi022 ceph-mon[24777]: from='client.24531 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:38:18.578 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:18 smithi022 ceph-mon[24777]: from='client.? 172.21.15.22:0/3621019191' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:38:18.655 INFO:teuthology.orchestra.run.smithi022.stdout:HEALTH_OK 2024-03-21T12:38:18.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:18 smithi189 ceph-mon[27080]: from='client.24531 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:38:18.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:18 smithi189 ceph-mon[27080]: from='client.? 172.21.15.22:0/3621019191' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:38:18.843 INFO:teuthology.orchestra.run.smithi022.stdout: 5 111993 13.21 MB/sec execute 78 sec latency 33.749 ms 2024-03-21T12:38:19.613 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:19 smithi022 ceph-mon[24777]: from='client.24539 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:38:19.614 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:19 smithi022 ceph-mon[24777]: pgmap v46: 97 pgs: 97 active+clean; 337 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.6 KiB/s rd, 6.4 MiB/s wr, 604 op/s 2024-03-21T12:38:19.614 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:19 smithi022 ceph-mon[24777]: from='client.? 172.21.15.22:0/1521927510' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-21T12:38:19.614 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:19 smithi022 ceph-mon[24777]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:38:19.614 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:19 smithi022 ceph-mon[24777]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:38:19.750 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:19 smithi189 ceph-mon[27080]: from='client.24539 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:38:19.750 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:19 smithi189 ceph-mon[27080]: pgmap v46: 97 pgs: 97 active+clean; 337 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.6 KiB/s rd, 6.4 MiB/s wr, 604 op/s 2024-03-21T12:38:19.750 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:19 smithi189 ceph-mon[27080]: from='client.? 172.21.15.22:0/1521927510' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-21T12:38:19.751 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:19 smithi189 ceph-mon[27080]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:38:19.751 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:19 smithi189 ceph-mon[27080]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:38:19.843 INFO:teuthology.orchestra.run.smithi022.stdout: 5 112427 13.23 MB/sec execute 79 sec latency 63.185 ms 2024-03-21T12:38:20.624 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:20 smithi022 ceph-mon[24777]: Reconfiguring node-exporter.smithi189 (dependencies changed)... 2024-03-21T12:38:20.624 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:20 smithi022 ceph-mon[24777]: Reconfiguring daemon node-exporter.smithi189 on smithi189 2024-03-21T12:38:20.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:20 smithi189 ceph-mon[27080]: Reconfiguring node-exporter.smithi189 (dependencies changed)... 2024-03-21T12:38:20.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:20 smithi189 ceph-mon[27080]: Reconfiguring daemon node-exporter.smithi189 on smithi189 2024-03-21T12:38:20.843 INFO:teuthology.orchestra.run.smithi022.stdout: 5 112962 13.20 MB/sec execute 80 sec latency 35.929 ms 2024-03-21T12:38:21.637 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:21 smithi189 ceph-mon[27080]: pgmap v47: 97 pgs: 97 active+clean; 326 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.4 MiB/s wr, 821 op/s 2024-03-21T12:38:21.640 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:21 smithi022 ceph-mon[24777]: pgmap v47: 97 pgs: 97 active+clean; 326 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.4 MiB/s wr, 821 op/s 2024-03-21T12:38:21.843 INFO:teuthology.orchestra.run.smithi022.stdout: 5 113419 13.22 MB/sec execute 81 sec latency 30.017 ms 2024-03-21T12:38:22.843 INFO:teuthology.orchestra.run.smithi022.stdout: 5 113817 13.19 MB/sec execute 82 sec latency 37.872 ms 2024-03-21T12:38:22.931 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:22 smithi022 ceph-mon[24777]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:38:22.931 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:22 smithi022 ceph-mon[24777]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:38:22.931 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:22 smithi022 ceph-mon[24777]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-21T12:38:22.931 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:22 smithi022 ceph-mon[24777]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-21T12:38:22.932 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:22 smithi022 ceph-mon[24777]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi022.front.sepia.ceph.com:9093"}]: dispatch 2024-03-21T12:38:22.932 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:22 smithi022 ceph-mon[24777]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi022.front.sepia.ceph.com:9093"}]: dispatch 2024-03-21T12:38:22.932 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:22 smithi022 ceph-mon[24777]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:38:22.932 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:22 smithi022 ceph-mon[24777]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-21T12:38:22.932 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:22 smithi022 ceph-mon[24777]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-21T12:38:22.932 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:22 smithi022 ceph-mon[24777]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi022.front.sepia.ceph.com:3000"}]: dispatch 2024-03-21T12:38:22.932 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:22 smithi022 ceph-mon[24777]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi022.front.sepia.ceph.com:3000"}]: dispatch 2024-03-21T12:38:22.932 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:22 smithi022 ceph-mon[24777]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:38:22.932 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:22 smithi022 ceph-mon[24777]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-21T12:38:22.932 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:22 smithi022 ceph-mon[24777]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-21T12:38:22.932 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:22 smithi022 ceph-mon[24777]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi022.front.sepia.ceph.com:9095"}]: dispatch 2024-03-21T12:38:22.932 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:22 smithi022 ceph-mon[24777]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi022.front.sepia.ceph.com:9095"}]: dispatch 2024-03-21T12:38:22.932 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:22 smithi022 ceph-mon[24777]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:38:22.932 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:22 smithi022 ceph-mon[24777]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:38:22.933 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:22 smithi022 ceph-mon[24777]: Upgrade: Need to upgrade myself (mgr.smithi189.cnxzvu) 2024-03-21T12:38:22.933 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:22 smithi022 ceph-mon[24777]: Upgrade: Need to upgrade myself (mgr.smithi189.cnxzvu) 2024-03-21T12:38:22.933 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:22 smithi022 ceph-mon[24777]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:38:23.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:22 smithi189 ceph-mon[27080]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:38:23.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:22 smithi189 ceph-mon[27080]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:38:23.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:22 smithi189 ceph-mon[27080]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-21T12:38:23.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:22 smithi189 ceph-mon[27080]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-21T12:38:23.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:22 smithi189 ceph-mon[27080]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi022.front.sepia.ceph.com:9093"}]: dispatch 2024-03-21T12:38:23.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:22 smithi189 ceph-mon[27080]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi022.front.sepia.ceph.com:9093"}]: dispatch 2024-03-21T12:38:23.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:22 smithi189 ceph-mon[27080]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:38:23.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:22 smithi189 ceph-mon[27080]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-21T12:38:23.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:22 smithi189 ceph-mon[27080]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-21T12:38:23.037 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:22 smithi189 ceph-mon[27080]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi022.front.sepia.ceph.com:3000"}]: dispatch 2024-03-21T12:38:23.037 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:22 smithi189 ceph-mon[27080]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi022.front.sepia.ceph.com:3000"}]: dispatch 2024-03-21T12:38:23.037 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:22 smithi189 ceph-mon[27080]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:38:23.037 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:22 smithi189 ceph-mon[27080]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-21T12:38:23.037 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:22 smithi189 ceph-mon[27080]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-21T12:38:23.037 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:22 smithi189 ceph-mon[27080]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi022.front.sepia.ceph.com:9095"}]: dispatch 2024-03-21T12:38:23.037 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:22 smithi189 ceph-mon[27080]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi022.front.sepia.ceph.com:9095"}]: dispatch 2024-03-21T12:38:23.037 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:22 smithi189 ceph-mon[27080]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:38:23.037 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:22 smithi189 ceph-mon[27080]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:38:23.037 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:22 smithi189 ceph-mon[27080]: Upgrade: Need to upgrade myself (mgr.smithi189.cnxzvu) 2024-03-21T12:38:23.037 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:22 smithi189 ceph-mon[27080]: Upgrade: Need to upgrade myself (mgr.smithi189.cnxzvu) 2024-03-21T12:38:23.037 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:22 smithi189 ceph-mon[27080]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:38:23.843 INFO:teuthology.orchestra.run.smithi022.stdout: 5 114210 13.23 MB/sec execute 83 sec latency 57.543 ms 2024-03-21T12:38:23.945 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:23 smithi022 ceph-mon[24777]: pgmap v48: 97 pgs: 97 active+clean; 326 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.6 MiB/s wr, 562 op/s 2024-03-21T12:38:24.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:23 smithi189 ceph-mon[27080]: pgmap v48: 97 pgs: 97 active+clean; 326 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.6 MiB/s wr, 562 op/s 2024-03-21T12:38:24.843 INFO:teuthology.orchestra.run.smithi022.stdout: 5 114542 13.14 MB/sec execute 84 sec latency 68.127 ms 2024-03-21T12:38:25.844 INFO:teuthology.orchestra.run.smithi022.stdout: 5 114965 13.04 MB/sec execute 85 sec latency 38.084 ms 2024-03-21T12:38:26.014 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:25 smithi022 ceph-mon[24777]: pgmap v49: 97 pgs: 97 active+clean; 332 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.3 MiB/s wr, 813 op/s 2024-03-21T12:38:26.014 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:25 smithi022 ceph-mon[24777]: mgrmap e27: smithi189.cnxzvu(active, since 92s), standbys: smithi022.fektzs 2024-03-21T12:38:26.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:25 smithi189 ceph-mon[27080]: pgmap v49: 97 pgs: 97 active+clean; 332 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.3 MiB/s wr, 813 op/s 2024-03-21T12:38:26.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:25 smithi189 ceph-mon[27080]: mgrmap e27: smithi189.cnxzvu(active, since 92s), standbys: smithi022.fektzs 2024-03-21T12:38:26.844 INFO:teuthology.orchestra.run.smithi022.stdout: 5 115328 13.05 MB/sec execute 86 sec latency 42.276 ms 2024-03-21T12:38:27.289 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:27 smithi022 ceph-mon[24777]: Upgrade: Updating mgr.smithi022.fektzs 2024-03-21T12:38:27.290 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:27 smithi022 ceph-mon[24777]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:38:27.290 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:27 smithi022 ceph-mon[24777]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi022.fektzs", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-21T12:38:27.290 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:27 smithi022 ceph-mon[24777]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi022.fektzs", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-21T12:38:27.290 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:27 smithi022 ceph-mon[24777]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-21T12:38:27.290 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:27 smithi022 ceph-mon[24777]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:38:27.290 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:27 smithi022 ceph-mon[24777]: Deploying daemon mgr.smithi022.fektzs on smithi022 2024-03-21T12:38:27.290 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:27 smithi022 ceph-mon[24777]: pgmap v50: 97 pgs: 97 active+clean; 332 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.7 MiB/s wr, 467 op/s 2024-03-21T12:38:27.344 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:27 smithi189 ceph-mon[27080]: Upgrade: Updating mgr.smithi022.fektzs 2024-03-21T12:38:27.344 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:27 smithi189 ceph-mon[27080]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:38:27.345 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:27 smithi189 ceph-mon[27080]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi022.fektzs", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-21T12:38:27.345 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:27 smithi189 ceph-mon[27080]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi022.fektzs", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-21T12:38:27.345 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:27 smithi189 ceph-mon[27080]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-21T12:38:27.345 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:27 smithi189 ceph-mon[27080]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:38:27.345 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:27 smithi189 ceph-mon[27080]: Deploying daemon mgr.smithi022.fektzs on smithi022 2024-03-21T12:38:27.345 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:27 smithi189 ceph-mon[27080]: pgmap v50: 97 pgs: 97 active+clean; 332 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.7 MiB/s wr, 467 op/s 2024-03-21T12:38:27.844 INFO:teuthology.orchestra.run.smithi022.stdout: 5 115782 13.07 MB/sec execute 87 sec latency 58.041 ms 2024-03-21T12:38:28.844 INFO:teuthology.orchestra.run.smithi022.stdout: 5 116420 13.05 MB/sec execute 88 sec latency 60.618 ms 2024-03-21T12:38:29.623 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:29 smithi022 ceph-mon[24777]: pgmap v51: 97 pgs: 97 active+clean; 332 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.7 MiB/s wr, 467 op/s 2024-03-21T12:38:29.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:29 smithi189 ceph-mon[27080]: pgmap v51: 97 pgs: 97 active+clean; 332 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.7 MiB/s wr, 467 op/s 2024-03-21T12:38:29.844 INFO:teuthology.orchestra.run.smithi022.stdout: 5 116958 13.09 MB/sec execute 89 sec latency 39.939 ms 2024-03-21T12:38:30.844 INFO:teuthology.orchestra.run.smithi022.stdout: 5 117419 13.10 MB/sec execute 90 sec latency 44.448 ms 2024-03-21T12:38:31.648 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:31 smithi022 ceph-mon[24777]: pgmap v52: 97 pgs: 97 active+clean; 326 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 7.0 MiB/s wr, 731 op/s 2024-03-21T12:38:31.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:31 smithi189 ceph-mon[27080]: pgmap v52: 97 pgs: 97 active+clean; 326 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 7.0 MiB/s wr, 731 op/s 2024-03-21T12:38:31.844 INFO:teuthology.orchestra.run.smithi022.stdout: 5 117921 13.17 MB/sec execute 91 sec latency 27.298 ms 2024-03-21T12:38:32.844 INFO:teuthology.orchestra.run.smithi022.stdout: 5 118360 13.07 MB/sec execute 92 sec latency 74.930 ms 2024-03-21T12:38:33.438 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:33 smithi022 ceph-mon[24777]: pgmap v53: 97 pgs: 97 active+clean; 326 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.0 MiB/s wr, 514 op/s 2024-03-21T12:38:33.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:33 smithi189 ceph-mon[27080]: pgmap v53: 97 pgs: 97 active+clean; 326 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.0 MiB/s wr, 514 op/s 2024-03-21T12:38:33.844 INFO:teuthology.orchestra.run.smithi022.stdout: 5 118641 12.97 MB/sec execute 93 sec latency 42.687 ms 2024-03-21T12:38:34.844 INFO:teuthology.orchestra.run.smithi022.stdout: 5 118984 12.98 MB/sec execute 94 sec latency 40.664 ms 2024-03-21T12:38:35.491 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:35 smithi022 ceph-mon[24777]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:38:35.491 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:35 smithi022 ceph-mon[24777]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:38:35.491 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:35 smithi022 ceph-mon[24777]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:38:35.491 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:35 smithi022 ceph-mon[24777]: pgmap v54: 97 pgs: 97 active+clean; 331 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 8.1 MiB/s wr, 791 op/s 2024-03-21T12:38:35.491 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:35 smithi022 ceph-mon[24777]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:38:35.491 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:35 smithi022 ceph-mon[24777]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:38:35.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:35 smithi189 ceph-mon[27080]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:38:35.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:35 smithi189 ceph-mon[27080]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:38:35.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:35 smithi189 ceph-mon[27080]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:38:35.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:35 smithi189 ceph-mon[27080]: pgmap v54: 97 pgs: 97 active+clean; 331 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 8.1 MiB/s wr, 791 op/s 2024-03-21T12:38:35.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:35 smithi189 ceph-mon[27080]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:38:35.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:35 smithi189 ceph-mon[27080]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:38:35.844 INFO:teuthology.orchestra.run.smithi022.stdout: 5 119674 13.07 MB/sec execute 95 sec latency 29.165 ms 2024-03-21T12:38:36.844 INFO:teuthology.orchestra.run.smithi022.stdout: 5 120227 13.00 MB/sec execute 96 sec latency 20.957 ms 2024-03-21T12:38:37.520 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:37 smithi022 ceph-mon[24777]: pgmap v55: 97 pgs: 97 active+clean; 331 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.4 MiB/s wr, 541 op/s 2024-03-21T12:38:37.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:37 smithi189 ceph-mon[27080]: pgmap v55: 97 pgs: 97 active+clean; 331 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.4 MiB/s wr, 541 op/s 2024-03-21T12:38:37.844 INFO:teuthology.orchestra.run.smithi022.stdout: 5 120664 13.05 MB/sec execute 97 sec latency 37.584 ms 2024-03-21T12:38:38.532 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:38 smithi022 ceph-mon[24777]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:38:38.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:38 smithi189 ceph-mon[27080]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:38:38.845 INFO:teuthology.orchestra.run.smithi022.stdout: 5 121224 13.16 MB/sec execute 98 sec latency 48.206 ms 2024-03-21T12:38:39.536 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:39 smithi022 ceph-mon[24777]: pgmap v56: 97 pgs: 97 active+clean; 331 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.4 MiB/s wr, 541 op/s 2024-03-21T12:38:39.537 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:39 smithi022 ceph-mon[24777]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:38:39.537 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:39 smithi022 ceph-mon[24777]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:38:39.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:39 smithi189 ceph-mon[27080]: pgmap v56: 97 pgs: 97 active+clean; 331 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.4 MiB/s wr, 541 op/s 2024-03-21T12:38:39.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:39 smithi189 ceph-mon[27080]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:38:39.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:39 smithi189 ceph-mon[27080]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:38:39.845 INFO:teuthology.orchestra.run.smithi022.stdout: 5 121680 13.08 MB/sec execute 99 sec latency 35.942 ms 2024-03-21T12:38:40.845 INFO:teuthology.orchestra.run.smithi022.stdout: 5 122023 13.00 MB/sec execute 100 sec latency 39.252 ms 2024-03-21T12:38:41.567 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:41 smithi022 ceph-mon[24777]: pgmap v57: 97 pgs: 97 active+clean; 335 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.6 MiB/s wr, 831 op/s 2024-03-21T12:38:41.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:41 smithi189 ceph-mon[27080]: pgmap v57: 97 pgs: 97 active+clean; 335 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.6 MiB/s wr, 831 op/s 2024-03-21T12:38:41.845 INFO:teuthology.orchestra.run.smithi022.stdout: 5 122406 12.99 MB/sec execute 101 sec latency 48.216 ms 2024-03-21T12:38:42.845 INFO:teuthology.orchestra.run.smithi022.stdout: 5 122984 13.09 MB/sec execute 102 sec latency 27.150 ms 2024-03-21T12:38:43.431 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:43 smithi022 ceph-mon[24777]: pgmap v58: 97 pgs: 97 active+clean; 335 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.8 KiB/s rd, 6.2 MiB/s wr, 567 op/s 2024-03-21T12:38:43.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:43 smithi189 ceph-mon[27080]: pgmap v58: 97 pgs: 97 active+clean; 335 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.8 KiB/s rd, 6.2 MiB/s wr, 567 op/s 2024-03-21T12:38:43.845 INFO:teuthology.orchestra.run.smithi022.stdout: 5 123432 13.00 MB/sec execute 103 sec latency 49.950 ms 2024-03-21T12:38:44.845 INFO:teuthology.orchestra.run.smithi022.stdout: 5 123821 12.93 MB/sec execute 104 sec latency 29.844 ms 2024-03-21T12:38:45.706 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:45 smithi022 ceph-mon[24777]: pgmap v59: 97 pgs: 97 active+clean; 328 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.8 MiB/s wr, 840 op/s 2024-03-21T12:38:45.706 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:45 smithi022 ceph-mon[24777]: Standby manager daemon smithi022.fektzs restarted 2024-03-21T12:38:45.706 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:45 smithi022 ceph-mon[24777]: Standby manager daemon smithi022.fektzs started 2024-03-21T12:38:45.706 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:45 smithi022 ceph-mon[24777]: from='mgr.? 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi022.fektzs/crt"}]: dispatch 2024-03-21T12:38:45.706 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:45 smithi022 ceph-mon[24777]: from='mgr.? 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-21T12:38:45.706 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:45 smithi022 ceph-mon[24777]: from='mgr.? 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi022.fektzs/key"}]: dispatch 2024-03-21T12:38:45.706 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:45 smithi022 ceph-mon[24777]: from='mgr.? 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-21T12:38:45.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:45 smithi189 ceph-mon[27080]: pgmap v59: 97 pgs: 97 active+clean; 328 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.8 MiB/s wr, 840 op/s 2024-03-21T12:38:45.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:45 smithi189 ceph-mon[27080]: Standby manager daemon smithi022.fektzs restarted 2024-03-21T12:38:45.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:45 smithi189 ceph-mon[27080]: Standby manager daemon smithi022.fektzs started 2024-03-21T12:38:45.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:45 smithi189 ceph-mon[27080]: from='mgr.? 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi022.fektzs/crt"}]: dispatch 2024-03-21T12:38:45.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:45 smithi189 ceph-mon[27080]: from='mgr.? 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-21T12:38:45.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:45 smithi189 ceph-mon[27080]: from='mgr.? 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi022.fektzs/key"}]: dispatch 2024-03-21T12:38:45.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:45 smithi189 ceph-mon[27080]: from='mgr.? 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-21T12:38:45.845 INFO:teuthology.orchestra.run.smithi022.stdout: 5 124228 12.97 MB/sec execute 105 sec latency 31.142 ms 2024-03-21T12:38:46.746 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:46 smithi022 ceph-mon[24777]: mgrmap e28: smithi189.cnxzvu(active, since 113s), standbys: smithi022.fektzs 2024-03-21T12:38:46.787 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:46 smithi189 ceph-mon[27080]: mgrmap e28: smithi189.cnxzvu(active, since 113s), standbys: smithi022.fektzs 2024-03-21T12:38:46.845 INFO:teuthology.orchestra.run.smithi022.stdout: 5 124686 13.01 MB/sec execute 106 sec latency 38.422 ms 2024-03-21T12:38:47.845 INFO:teuthology.orchestra.run.smithi022.stdout: 5 125165 12.99 MB/sec execute 107 sec latency 77.796 ms 2024-03-21T12:38:48.019 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:47 smithi022 ceph-mon[24777]: pgmap v60: 97 pgs: 97 active+clean; 328 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.8 MiB/s wr, 562 op/s 2024-03-21T12:38:48.020 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:47 smithi022 ceph-mon[24777]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:38:48.020 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:47 smithi022 ceph-mon[24777]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:38:48.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:47 smithi189 ceph-mon[27080]: pgmap v60: 97 pgs: 97 active+clean; 328 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.8 MiB/s wr, 562 op/s 2024-03-21T12:38:48.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:47 smithi189 ceph-mon[27080]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:38:48.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:47 smithi189 ceph-mon[27080]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:38:48.845 INFO:teuthology.orchestra.run.smithi022.stdout: 5 125578 12.93 MB/sec execute 108 sec latency 27.120 ms 2024-03-21T12:38:49.208 INFO:teuthology.orchestra.run.smithi022.stdout:true 2024-03-21T12:38:49.295 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:49 smithi022 ceph-mon[24777]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:38:49.295 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:49 smithi022 ceph-mon[24777]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:38:49.296 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:49 smithi022 ceph-mon[24777]: pgmap v61: 97 pgs: 97 active+clean; 328 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.8 MiB/s wr, 562 op/s 2024-03-21T12:38:49.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:49 smithi189 ceph-mon[27080]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:38:49.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:49 smithi189 ceph-mon[27080]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:38:49.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:49 smithi189 ceph-mon[27080]: pgmap v61: 97 pgs: 97 active+clean; 328 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.8 MiB/s wr, 562 op/s 2024-03-21T12:38:49.845 INFO:teuthology.orchestra.run.smithi022.stdout: 5 126046 12.93 MB/sec execute 109 sec latency 35.902 ms 2024-03-21T12:38:50.075 INFO:teuthology.orchestra.run.smithi022.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-21T12:38:50.075 INFO:teuthology.orchestra.run.smithi022.stdout:alertmanager.smithi022 smithi022 *:9093,9094 running (81s) 3s ago 10m 25.7M - 0.23.0 ba2b418f427c 14eb2a3ca318 2024-03-21T12:38:50.076 INFO:teuthology.orchestra.run.smithi022.stdout:crash.smithi022 smithi022 running (10m) 3s ago 10m 6405k - 17.2.0 e1d6a67b021e 47deb70fd250 2024-03-21T12:38:50.076 INFO:teuthology.orchestra.run.smithi022.stdout:crash.smithi189 smithi189 running (9m) 11s ago 9m 6405k - 17.2.0 e1d6a67b021e 9a2e0e9b1809 2024-03-21T12:38:50.076 INFO:teuthology.orchestra.run.smithi022.stdout:grafana.smithi022 smithi022 *:3000 unknown 3s ago 9m - - 2024-03-21T12:38:50.076 INFO:teuthology.orchestra.run.smithi022.stdout:mds.foofs.smithi022.bcnxho smithi022 running (4m) 3s ago 4m 36.7M - 17.2.0 e1d6a67b021e db26b67b8ee7 2024-03-21T12:38:50.076 INFO:teuthology.orchestra.run.smithi022.stdout:mds.foofs.smithi189.lluvko smithi189 running (4m) 11s ago 4m 15.9M - 17.2.0 e1d6a67b021e cf153aff1387 2024-03-21T12:38:50.076 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.smithi022.fektzs smithi022 *:8443,9283,8765 running (16s) 3s ago 11m 89.3M - 19.0.0-2267-g960fbbe4 19e12293512f d57cebd306b3 2024-03-21T12:38:50.076 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.smithi189.cnxzvu smithi189 *:8443,9283 running (2m) 11s ago 9m 506M - 19.0.0-2267-g960fbbe4 19e12293512f 07f3cf369ff7 2024-03-21T12:38:50.076 INFO:teuthology.orchestra.run.smithi022.stdout:mon.smithi022 smithi022 running (11m) 3s ago 11m 51.9M 2048M 17.2.0 e1d6a67b021e 0d248775e0e5 2024-03-21T12:38:50.076 INFO:teuthology.orchestra.run.smithi022.stdout:mon.smithi189 smithi189 running (9m) 11s ago 9m 42.1M 2048M 17.2.0 e1d6a67b021e 82da5f3cd49a 2024-03-21T12:38:50.076 INFO:teuthology.orchestra.run.smithi022.stdout:nfs.foo.0.0.smithi022.ynhbbq smithi022 *:2049 running (4m) 3s ago 4m 258M - 4.0 e1d6a67b021e 33e48e64c60b 2024-03-21T12:38:50.076 INFO:teuthology.orchestra.run.smithi022.stdout:nfs.foo.1.0.smithi189.yzsqml smithi189 *:2049 running (3m) 11s ago 3m 79.2M - 4.0 e1d6a67b021e 48fab2cbdde2 2024-03-21T12:38:50.076 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.smithi022 smithi022 *:9100 running (57s) 3s ago 9m 8544k - 1.3.1 1dbe0e931976 33287f803f4f 2024-03-21T12:38:50.076 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.smithi189 smithi189 *:9100 running (28s) 11s ago 9m 8120k - 1.3.1 1dbe0e931976 b51d872e438a 2024-03-21T12:38:50.076 INFO:teuthology.orchestra.run.smithi022.stdout:osd.0 smithi022 running (8m) 3s ago 8m 445M 1937M 17.2.0 e1d6a67b021e 54e6631bd10f 2024-03-21T12:38:50.077 INFO:teuthology.orchestra.run.smithi022.stdout:osd.1 smithi022 running (8m) 3s ago 8m 377M 1937M 17.2.0 e1d6a67b021e 68165c5d09cb 2024-03-21T12:38:50.077 INFO:teuthology.orchestra.run.smithi022.stdout:osd.2 smithi022 running (7m) 3s ago 7m 269M 1937M 17.2.0 e1d6a67b021e fbec9bef8433 2024-03-21T12:38:50.077 INFO:teuthology.orchestra.run.smithi022.stdout:osd.3 smithi022 running (7m) 3s ago 7m 411M 1937M 17.2.0 e1d6a67b021e b1462ffc1b9e 2024-03-21T12:38:50.077 INFO:teuthology.orchestra.run.smithi022.stdout:osd.4 smithi189 running (7m) 11s ago 7m 394M 2704M 17.2.0 e1d6a67b021e 1bedc4eb02ed 2024-03-21T12:38:50.077 INFO:teuthology.orchestra.run.smithi022.stdout:osd.5 smithi189 running (6m) 11s ago 6m 451M 2704M 17.2.0 e1d6a67b021e 25fbab9786a5 2024-03-21T12:38:50.077 INFO:teuthology.orchestra.run.smithi022.stdout:osd.6 smithi189 running (6m) 11s ago 6m 382M 2704M 17.2.0 e1d6a67b021e 0c192fcfd65b 2024-03-21T12:38:50.077 INFO:teuthology.orchestra.run.smithi022.stdout:osd.7 smithi189 running (5m) 11s ago 5m 536M 2704M 17.2.0 e1d6a67b021e 3d0723e1f863 2024-03-21T12:38:50.077 INFO:teuthology.orchestra.run.smithi022.stdout:prometheus.smithi022 smithi022 *:9095 running (32s) 3s ago 9m 51.0M - 2.33.4 514e6a882f6e f632d068e116 2024-03-21T12:38:50.319 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:50 smithi022 ceph-mon[24777]: from='client.14772 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:38:50.319 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:50 smithi022 ceph-mon[24777]: from='client.14776 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:38:50.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:50 smithi189 ceph-mon[27080]: from='client.14772 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:38:50.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:50 smithi189 ceph-mon[27080]: from='client.14776 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:38:50.628 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-03-21T12:38:50.628 INFO:teuthology.orchestra.run.smithi022.stdout: "mon": { 2024-03-21T12:38:50.628 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-21T12:38:50.628 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:38:50.628 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr": { 2024-03-21T12:38:50.628 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 2 2024-03-21T12:38:50.628 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:38:50.628 INFO:teuthology.orchestra.run.smithi022.stdout: "osd": { 2024-03-21T12:38:50.629 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-03-21T12:38:50.629 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:38:50.629 INFO:teuthology.orchestra.run.smithi022.stdout: "mds": { 2024-03-21T12:38:50.629 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-21T12:38:50.629 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:38:50.629 INFO:teuthology.orchestra.run.smithi022.stdout: "overall": { 2024-03-21T12:38:50.629 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-03-21T12:38:50.629 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 2 2024-03-21T12:38:50.629 INFO:teuthology.orchestra.run.smithi022.stdout: } 2024-03-21T12:38:50.629 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-03-21T12:38:50.845 INFO:teuthology.orchestra.run.smithi022.stdout: 5 126419 12.95 MB/sec execute 110 sec latency 36.287 ms 2024-03-21T12:38:51.059 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-03-21T12:38:51.059 INFO:teuthology.orchestra.run.smithi022.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93", 2024-03-21T12:38:51.059 INFO:teuthology.orchestra.run.smithi022.stdout: "in_progress": true, 2024-03-21T12:38:51.059 INFO:teuthology.orchestra.run.smithi022.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-21T12:38:51.059 INFO:teuthology.orchestra.run.smithi022.stdout: "services_complete": [ 2024-03-21T12:38:51.059 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr" 2024-03-21T12:38:51.059 INFO:teuthology.orchestra.run.smithi022.stdout: ], 2024-03-21T12:38:51.059 INFO:teuthology.orchestra.run.smithi022.stdout: "progress": "2/23 daemons upgraded", 2024-03-21T12:38:51.059 INFO:teuthology.orchestra.run.smithi022.stdout: "message": "Currently upgrading mgr daemons", 2024-03-21T12:38:51.059 INFO:teuthology.orchestra.run.smithi022.stdout: "is_paused": false 2024-03-21T12:38:51.059 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-03-21T12:38:51.332 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:51 smithi022 ceph-mon[24777]: from='client.14780 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:38:51.332 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:51 smithi022 ceph-mon[24777]: pgmap v62: 97 pgs: 97 active+clean; 330 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.5 MiB/s wr, 821 op/s 2024-03-21T12:38:51.332 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:51 smithi022 ceph-mon[24777]: from='client.? 172.21.15.22:0/3004162972' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:38:51.607 INFO:teuthology.orchestra.run.smithi022.stdout:HEALTH_OK 2024-03-21T12:38:51.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:51 smithi189 ceph-mon[27080]: from='client.14780 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:38:51.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:51 smithi189 ceph-mon[27080]: pgmap v62: 97 pgs: 97 active+clean; 330 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.5 MiB/s wr, 821 op/s 2024-03-21T12:38:51.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:51 smithi189 ceph-mon[27080]: from='client.? 172.21.15.22:0/3004162972' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:38:51.845 INFO:teuthology.orchestra.run.smithi022.stdout: 5 126767 12.91 MB/sec execute 111 sec latency 51.278 ms 2024-03-21T12:38:52.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:52 smithi022 ceph-mon[24777]: from='client.14788 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:38:52.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:52 smithi022 ceph-mon[24777]: from='client.? 172.21.15.22:0/2983655485' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-21T12:38:52.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:52 smithi189 ceph-mon[27080]: from='client.14788 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:38:52.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:52 smithi189 ceph-mon[27080]: from='client.? 172.21.15.22:0/2983655485' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-21T12:38:52.845 INFO:teuthology.orchestra.run.smithi022.stdout: 5 127133 12.83 MB/sec execute 112 sec latency 61.780 ms 2024-03-21T12:38:53.370 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:53 smithi022 ceph-mon[24777]: pgmap v63: 97 pgs: 97 active+clean; 330 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.3 MiB/s wr, 531 op/s 2024-03-21T12:38:53.370 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:53 smithi022 ceph-mon[24777]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:38:53.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:53 smithi189 ceph-mon[27080]: pgmap v63: 97 pgs: 97 active+clean; 330 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.3 MiB/s wr, 531 op/s 2024-03-21T12:38:53.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:53 smithi189 ceph-mon[27080]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:38:53.845 INFO:teuthology.orchestra.run.smithi022.stdout: 5 127416 12.76 MB/sec execute 113 sec latency 62.141 ms 2024-03-21T12:38:54.846 INFO:teuthology.orchestra.run.smithi022.stdout: 5 127780 12.78 MB/sec execute 114 sec latency 42.524 ms 2024-03-21T12:38:55.412 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:55 smithi022 ceph-mon[24777]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:38:55.412 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:55 smithi022 ceph-mon[24777]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:38:55.412 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:55 smithi022 ceph-mon[24777]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:38:55.412 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:55 smithi022 ceph-mon[24777]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:38:55.412 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:55 smithi022 ceph-mon[24777]: pgmap v64: 97 pgs: 97 active+clean; 328 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.7 MiB/s wr, 779 op/s 2024-03-21T12:38:55.412 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:55 smithi022 ceph-mon[24777]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:38:55.412 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:55 smithi022 ceph-mon[24777]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:38:55.412 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:55 smithi022 ceph-mon[24777]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:38:55.412 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:55 smithi022 ceph-mon[24777]: Upgrade: Need to upgrade myself (mgr.smithi189.cnxzvu) 2024-03-21T12:38:55.412 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:55 smithi022 ceph-mon[24777]: Upgrade: Need to upgrade myself (mgr.smithi189.cnxzvu) 2024-03-21T12:38:55.412 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:55 smithi022 ceph-mon[24777]: Failing over to other MGR 2024-03-21T12:38:55.413 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:55 smithi022 ceph-mon[24777]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "mgr fail", "who": "smithi189.cnxzvu"}]: dispatch 2024-03-21T12:38:55.413 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:55 smithi022 ceph-mon[24777]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "mgr fail", "who": "smithi189.cnxzvu"}]: dispatch 2024-03-21T12:38:55.413 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:55 smithi022 ceph-mon[24777]: osdmap e72: 8 total, 8 up, 8 in 2024-03-21T12:38:55.413 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:55 smithi022 ceph-mon[24777]: Standby manager daemon smithi189.cnxzvu started 2024-03-21T12:38:55.504 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:55 smithi189 ceph-mon[27080]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:38:55.504 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:55 smithi189 ceph-mon[27080]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:38:55.505 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:55 smithi189 ceph-mon[27080]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:38:55.505 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:55 smithi189 ceph-mon[27080]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:38:55.505 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:55 smithi189 ceph-mon[27080]: pgmap v64: 97 pgs: 97 active+clean; 328 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.7 MiB/s wr, 779 op/s 2024-03-21T12:38:55.505 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:55 smithi189 ceph-mon[27080]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:38:55.505 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:55 smithi189 ceph-mon[27080]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' 2024-03-21T12:38:55.505 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:55 smithi189 ceph-mon[27080]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:38:55.505 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:55 smithi189 ceph-mon[27080]: Upgrade: Need to upgrade myself (mgr.smithi189.cnxzvu) 2024-03-21T12:38:55.505 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:55 smithi189 ceph-mon[27080]: Upgrade: Need to upgrade myself (mgr.smithi189.cnxzvu) 2024-03-21T12:38:55.505 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:55 smithi189 ceph-mon[27080]: Failing over to other MGR 2024-03-21T12:38:55.505 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:55 smithi189 ceph-mon[27080]: from='mgr.24453 172.21.15.189:0/2196062843' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "mgr fail", "who": "smithi189.cnxzvu"}]: dispatch 2024-03-21T12:38:55.505 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:55 smithi189 ceph-mon[27080]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "mgr fail", "who": "smithi189.cnxzvu"}]: dispatch 2024-03-21T12:38:55.505 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:55 smithi189 ceph-mon[27080]: osdmap e72: 8 total, 8 up, 8 in 2024-03-21T12:38:55.505 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:55 smithi189 ceph-mon[27080]: Standby manager daemon smithi189.cnxzvu started 2024-03-21T12:38:55.846 INFO:teuthology.orchestra.run.smithi022.stdout: 5 128022 12.72 MB/sec execute 115 sec latency 59.840 ms 2024-03-21T12:38:56.423 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:56 smithi022 ceph-mon[24777]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-03-21T12:38:56.423 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:56 smithi022 ceph-mon[24777]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' cmd='[{"prefix": "mgr fail", "who": "smithi189.cnxzvu"}]': finished 2024-03-21T12:38:56.423 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:56 smithi022 ceph-mon[24777]: mgrmap e29: smithi022.fektzs(active, starting, since 0.867356s), standbys: smithi189.cnxzvu 2024-03-21T12:38:56.423 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:56 smithi022 ceph-mon[24777]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mon metadata", "id": "smithi022"}]: dispatch 2024-03-21T12:38:56.423 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:56 smithi022 ceph-mon[24777]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mon metadata", "id": "smithi189"}]: dispatch 2024-03-21T12:38:56.423 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:56 smithi022 ceph-mon[24777]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi022.bcnxho"}]: dispatch 2024-03-21T12:38:56.423 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:56 smithi022 ceph-mon[24777]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi189.lluvko"}]: dispatch 2024-03-21T12:38:56.424 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:56 smithi022 ceph-mon[24777]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mgr metadata", "who": "smithi022.fektzs", "id": "smithi022.fektzs"}]: dispatch 2024-03-21T12:38:56.424 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:56 smithi022 ceph-mon[24777]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mgr metadata", "who": "smithi189.cnxzvu", "id": "smithi189.cnxzvu"}]: dispatch 2024-03-21T12:38:56.424 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:56 smithi022 ceph-mon[24777]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-21T12:38:56.424 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:56 smithi022 ceph-mon[24777]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-21T12:38:56.424 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:56 smithi022 ceph-mon[24777]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-21T12:38:56.424 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:56 smithi022 ceph-mon[24777]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-21T12:38:56.424 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:56 smithi022 ceph-mon[24777]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-21T12:38:56.424 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:56 smithi022 ceph-mon[24777]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-21T12:38:56.424 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:56 smithi022 ceph-mon[24777]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-21T12:38:56.424 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:56 smithi022 ceph-mon[24777]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-21T12:38:56.424 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:56 smithi022 ceph-mon[24777]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-21T12:38:56.424 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:56 smithi022 ceph-mon[24777]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-21T12:38:56.424 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:56 smithi022 ceph-mon[24777]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-21T12:38:56.424 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:56 smithi022 ceph-mon[24777]: Manager daemon smithi022.fektzs is now available 2024-03-21T12:38:56.424 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:56 smithi022 ceph-mon[24777]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' 2024-03-21T12:38:56.424 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:56 smithi022 ceph-mon[24777]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:38:56.425 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:56 smithi022 ceph-mon[24777]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:38:56.425 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:56 smithi022 ceph-mon[24777]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi022.fektzs/mirror_snapshot_schedule"}]: dispatch 2024-03-21T12:38:56.425 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:56 smithi022 ceph-mon[24777]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi022.fektzs/trash_purge_schedule"}]: dispatch 2024-03-21T12:38:56.425 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:56 smithi022 ceph-mon[24777]: [21/Mar/2024:12:38:55] ENGINE Bus STARTING 2024-03-21T12:38:56.425 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:56 smithi022 ceph-mon[24777]: [21/Mar/2024:12:38:55] ENGINE Serving on https://172.21.15.22:7150 2024-03-21T12:38:56.425 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:56 smithi022 ceph-mon[24777]: [21/Mar/2024:12:38:56] ENGINE Serving on http://172.21.15.22:8765 2024-03-21T12:38:56.425 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:56 smithi022 ceph-mon[24777]: [21/Mar/2024:12:38:56] ENGINE Bus STARTED 2024-03-21T12:38:56.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:56 smithi189 ceph-mon[27080]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-03-21T12:38:56.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:56 smithi189 ceph-mon[27080]: from='mgr.24453 ' entity='mgr.smithi189.cnxzvu' cmd='[{"prefix": "mgr fail", "who": "smithi189.cnxzvu"}]': finished 2024-03-21T12:38:56.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:56 smithi189 ceph-mon[27080]: mgrmap e29: smithi022.fektzs(active, starting, since 0.867356s), standbys: smithi189.cnxzvu 2024-03-21T12:38:56.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:56 smithi189 ceph-mon[27080]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mon metadata", "id": "smithi022"}]: dispatch 2024-03-21T12:38:56.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:56 smithi189 ceph-mon[27080]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mon metadata", "id": "smithi189"}]: dispatch 2024-03-21T12:38:56.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:56 smithi189 ceph-mon[27080]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi022.bcnxho"}]: dispatch 2024-03-21T12:38:56.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:56 smithi189 ceph-mon[27080]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi189.lluvko"}]: dispatch 2024-03-21T12:38:56.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:56 smithi189 ceph-mon[27080]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mgr metadata", "who": "smithi022.fektzs", "id": "smithi022.fektzs"}]: dispatch 2024-03-21T12:38:56.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:56 smithi189 ceph-mon[27080]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mgr metadata", "who": "smithi189.cnxzvu", "id": "smithi189.cnxzvu"}]: dispatch 2024-03-21T12:38:56.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:56 smithi189 ceph-mon[27080]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-21T12:38:56.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:56 smithi189 ceph-mon[27080]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-21T12:38:56.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:56 smithi189 ceph-mon[27080]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-21T12:38:56.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:56 smithi189 ceph-mon[27080]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-21T12:38:56.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:56 smithi189 ceph-mon[27080]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-21T12:38:56.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:56 smithi189 ceph-mon[27080]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-21T12:38:56.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:56 smithi189 ceph-mon[27080]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-21T12:38:56.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:56 smithi189 ceph-mon[27080]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-21T12:38:56.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:56 smithi189 ceph-mon[27080]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-21T12:38:56.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:56 smithi189 ceph-mon[27080]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-21T12:38:56.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:56 smithi189 ceph-mon[27080]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-21T12:38:56.538 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:56 smithi189 ceph-mon[27080]: Manager daemon smithi022.fektzs is now available 2024-03-21T12:38:56.538 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:56 smithi189 ceph-mon[27080]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' 2024-03-21T12:38:56.538 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:56 smithi189 ceph-mon[27080]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:38:56.538 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:56 smithi189 ceph-mon[27080]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:38:56.538 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:56 smithi189 ceph-mon[27080]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi022.fektzs/mirror_snapshot_schedule"}]: dispatch 2024-03-21T12:38:56.538 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:56 smithi189 ceph-mon[27080]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi022.fektzs/trash_purge_schedule"}]: dispatch 2024-03-21T12:38:56.538 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:56 smithi189 ceph-mon[27080]: [21/Mar/2024:12:38:55] ENGINE Bus STARTING 2024-03-21T12:38:56.538 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:56 smithi189 ceph-mon[27080]: [21/Mar/2024:12:38:55] ENGINE Serving on https://172.21.15.22:7150 2024-03-21T12:38:56.538 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:56 smithi189 ceph-mon[27080]: [21/Mar/2024:12:38:56] ENGINE Serving on http://172.21.15.22:8765 2024-03-21T12:38:56.538 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:56 smithi189 ceph-mon[27080]: [21/Mar/2024:12:38:56] ENGINE Bus STARTED 2024-03-21T12:38:56.846 INFO:teuthology.orchestra.run.smithi022.stdout: 5 128365 12.74 MB/sec execute 116 sec latency 95.188 ms 2024-03-21T12:38:57.449 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:57 smithi022 ceph-mon[24777]: mgrmap e30: smithi022.fektzs(active, since 1.86837s), standbys: smithi189.cnxzvu 2024-03-21T12:38:57.449 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:57 smithi022 ceph-mon[24777]: pgmap v3: 97 pgs: 97 active+clean; 331 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-03-21T12:38:57.468 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:57 smithi189 ceph-mon[27080]: mgrmap e30: smithi022.fektzs(active, since 1.86837s), standbys: smithi189.cnxzvu 2024-03-21T12:38:57.468 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:57 smithi189 ceph-mon[27080]: pgmap v3: 97 pgs: 97 active+clean; 331 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-03-21T12:38:57.846 INFO:teuthology.orchestra.run.smithi022.stdout: 5 128661 12.69 MB/sec execute 117 sec latency 107.478 ms 2024-03-21T12:38:58.473 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:58 smithi022 ceph-mon[24777]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-03-21T12:38:58.473 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:58 smithi022 ceph-mon[24777]: Cluster is now healthy 2024-03-21T12:38:58.473 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:58 smithi022 ceph-mon[24777]: mgrmap e31: smithi022.fektzs(active, since 2s), standbys: smithi189.cnxzvu 2024-03-21T12:38:58.473 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:38:58 smithi022 ceph-mon[24777]: pgmap v4: 97 pgs: 97 active+clean; 331 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-03-21T12:38:58.478 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:58 smithi189 ceph-mon[27080]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-03-21T12:38:58.478 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:58 smithi189 ceph-mon[27080]: Cluster is now healthy 2024-03-21T12:38:58.478 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:58 smithi189 ceph-mon[27080]: mgrmap e31: smithi022.fektzs(active, since 2s), standbys: smithi189.cnxzvu 2024-03-21T12:38:58.478 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:38:58 smithi189 ceph-mon[27080]: pgmap v4: 97 pgs: 97 active+clean; 331 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-03-21T12:38:58.846 INFO:teuthology.orchestra.run.smithi022.stdout: 5 128898 12.62 MB/sec execute 118 sec latency 60.506 ms 2024-03-21T12:38:59.846 INFO:teuthology.orchestra.run.smithi022.stdout: 5 129095 12.53 MB/sec execute 119 sec latency 69.993 ms 2024-03-21T12:39:00.248 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:00 smithi022 ceph-mon[24777]: pgmap v5: 97 pgs: 97 active+clean; 331 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-03-21T12:39:00.249 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:00 smithi022 ceph-mon[24777]: mgrmap e32: smithi022.fektzs(active, since 4s), standbys: smithi189.cnxzvu 2024-03-21T12:39:00.428 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:00 smithi189 ceph-mon[27080]: pgmap v5: 97 pgs: 97 active+clean; 331 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-03-21T12:39:00.428 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:00 smithi189 ceph-mon[27080]: mgrmap e32: smithi022.fektzs(active, since 4s), standbys: smithi189.cnxzvu 2024-03-21T12:39:00.846 INFO:teuthology.orchestra.run.smithi022.stdout: 5 129585 12.54 MB/sec execute 120 sec latency 56.316 ms 2024-03-21T12:39:01.519 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:01 smithi022 ceph-mon[24777]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' 2024-03-21T12:39:01.519 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:01 smithi022 ceph-mon[24777]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' 2024-03-21T12:39:01.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:01 smithi189 ceph-mon[27080]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' 2024-03-21T12:39:01.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:01 smithi189 ceph-mon[27080]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' 2024-03-21T12:39:01.846 INFO:teuthology.orchestra.run.smithi022.stdout: 5 130023 12.59 MB/sec execute 121 sec latency 36.241 ms 2024-03-21T12:39:02.776 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:02 smithi022 ceph-mon[24777]: pgmap v6: 97 pgs: 97 active+clean; 331 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.1 KiB/s rd, 5.1 KiB/s wr, 4 op/s 2024-03-21T12:39:02.777 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:02 smithi022 ceph-mon[24777]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' 2024-03-21T12:39:02.777 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:02 smithi022 ceph-mon[24777]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' 2024-03-21T12:39:02.846 INFO:teuthology.orchestra.run.smithi022.stdout: 5 130437 12.54 MB/sec execute 122 sec latency 47.613 ms 2024-03-21T12:39:03.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:02 smithi189 ceph-mon[27080]: pgmap v6: 97 pgs: 97 active+clean; 331 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.1 KiB/s rd, 5.1 KiB/s wr, 4 op/s 2024-03-21T12:39:03.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:02 smithi189 ceph-mon[27080]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' 2024-03-21T12:39:03.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:02 smithi189 ceph-mon[27080]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' 2024-03-21T12:39:03.846 INFO:teuthology.orchestra.run.smithi022.stdout: 5 130740 12.47 MB/sec execute 123 sec latency 43.506 ms 2024-03-21T12:39:04.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:04 smithi189 ceph-mon[27080]: pgmap v7: 97 pgs: 97 active+clean; 344 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 33 KiB/s rd, 3.5 MiB/s wr, 347 op/s 2024-03-21T12:39:04.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:04 smithi189 ceph-mon[27080]: Standby manager daemon smithi189.cnxzvu restarted 2024-03-21T12:39:04.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:04 smithi189 ceph-mon[27080]: Standby manager daemon smithi189.cnxzvu started 2024-03-21T12:39:04.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:04 smithi189 ceph-mon[27080]: from='mgr.? 172.21.15.189:0/4161773458' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi189.cnxzvu/crt"}]: dispatch 2024-03-21T12:39:04.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:04 smithi189 ceph-mon[27080]: from='mgr.? 172.21.15.189:0/4161773458' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-21T12:39:04.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:04 smithi189 ceph-mon[27080]: from='mgr.? 172.21.15.189:0/4161773458' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi189.cnxzvu/key"}]: dispatch 2024-03-21T12:39:04.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:04 smithi189 ceph-mon[27080]: from='mgr.? 172.21.15.189:0/4161773458' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-21T12:39:04.821 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:04 smithi022 ceph-mon[24777]: pgmap v7: 97 pgs: 97 active+clean; 344 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 33 KiB/s rd, 3.5 MiB/s wr, 347 op/s 2024-03-21T12:39:04.822 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:04 smithi022 ceph-mon[24777]: Standby manager daemon smithi189.cnxzvu restarted 2024-03-21T12:39:04.822 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:04 smithi022 ceph-mon[24777]: Standby manager daemon smithi189.cnxzvu started 2024-03-21T12:39:04.822 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:04 smithi022 ceph-mon[24777]: from='mgr.? 172.21.15.189:0/4161773458' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi189.cnxzvu/crt"}]: dispatch 2024-03-21T12:39:04.822 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:04 smithi022 ceph-mon[24777]: from='mgr.? 172.21.15.189:0/4161773458' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-21T12:39:04.822 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:04 smithi022 ceph-mon[24777]: from='mgr.? 172.21.15.189:0/4161773458' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi189.cnxzvu/key"}]: dispatch 2024-03-21T12:39:04.822 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:04 smithi022 ceph-mon[24777]: from='mgr.? 172.21.15.189:0/4161773458' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-21T12:39:04.846 INFO:teuthology.orchestra.run.smithi022.stdout: 5 130971 12.40 MB/sec execute 124 sec latency 49.246 ms 2024-03-21T12:39:05.835 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:05 smithi022 ceph-mon[24777]: mgrmap e33: smithi022.fektzs(active, since 10s), standbys: smithi189.cnxzvu 2024-03-21T12:39:05.835 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:05 smithi022 ceph-mon[24777]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' 2024-03-21T12:39:05.835 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:05 smithi022 ceph-mon[24777]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' 2024-03-21T12:39:05.846 INFO:teuthology.orchestra.run.smithi022.stdout: 5 131297 12.42 MB/sec execute 125 sec latency 32.641 ms 2024-03-21T12:39:06.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:05 smithi189 ceph-mon[27080]: mgrmap e33: smithi022.fektzs(active, since 10s), standbys: smithi189.cnxzvu 2024-03-21T12:39:06.068 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:05 smithi189 ceph-mon[27080]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' 2024-03-21T12:39:06.068 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:05 smithi189 ceph-mon[27080]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' 2024-03-21T12:39:06.846 INFO:teuthology.orchestra.run.smithi022.stdout: 5 131653 12.40 MB/sec execute 126 sec latency 48.556 ms 2024-03-21T12:39:07.102 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:07 smithi022 ceph-mon[24777]: pgmap v8: 97 pgs: 97 active+clean; 344 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 26 KiB/s rd, 2.7 MiB/s wr, 269 op/s 2024-03-21T12:39:07.102 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:07 smithi022 ceph-mon[24777]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' 2024-03-21T12:39:07.102 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:07 smithi022 ceph-mon[24777]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' 2024-03-21T12:39:07.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:07 smithi189 ceph-mon[27080]: pgmap v8: 97 pgs: 97 active+clean; 344 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 26 KiB/s rd, 2.7 MiB/s wr, 269 op/s 2024-03-21T12:39:07.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:07 smithi189 ceph-mon[27080]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' 2024-03-21T12:39:07.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:07 smithi189 ceph-mon[27080]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' 2024-03-21T12:39:07.846 INFO:teuthology.orchestra.run.smithi022.stdout: 5 132092 12.42 MB/sec execute 127 sec latency 61.241 ms 2024-03-21T12:39:08.846 INFO:teuthology.orchestra.run.smithi022.stdout: 5 132400 12.37 MB/sec execute 128 sec latency 43.460 ms 2024-03-21T12:39:09.147 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:09 smithi022 ceph-mon[24777]: pgmap v9: 97 pgs: 97 active+clean; 336 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 25 KiB/s rd, 4.4 MiB/s wr, 451 op/s 2024-03-21T12:39:09.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:09 smithi189 ceph-mon[27080]: pgmap v9: 97 pgs: 97 active+clean; 336 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 25 KiB/s rd, 4.4 MiB/s wr, 451 op/s 2024-03-21T12:39:09.846 INFO:teuthology.orchestra.run.smithi022.stdout: 5 132667 12.30 MB/sec execute 129 sec latency 40.433 ms 2024-03-21T12:39:10.846 INFO:teuthology.orchestra.run.smithi022.stdout: 5 133038 12.30 MB/sec execute 130 sec latency 39.149 ms 2024-03-21T12:39:11.194 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:11 smithi022 ceph-mon[24777]: pgmap v10: 97 pgs: 97 active+clean; 336 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 23 KiB/s rd, 4.0 MiB/s wr, 410 op/s 2024-03-21T12:39:11.194 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:11 smithi022 ceph-mon[24777]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' 2024-03-21T12:39:11.195 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:11 smithi022 ceph-mon[24777]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:39:11.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:11 smithi189 ceph-mon[27080]: pgmap v10: 97 pgs: 97 active+clean; 336 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 23 KiB/s rd, 4.0 MiB/s wr, 410 op/s 2024-03-21T12:39:11.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:11 smithi189 ceph-mon[27080]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' 2024-03-21T12:39:11.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:11 smithi189 ceph-mon[27080]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:39:11.846 INFO:teuthology.orchestra.run.smithi022.stdout: 5 133366 12.30 MB/sec execute 131 sec latency 36.240 ms 2024-03-21T12:39:12.847 INFO:teuthology.orchestra.run.smithi022.stdout: 5 133716 12.30 MB/sec execute 132 sec latency 44.293 ms 2024-03-21T12:39:13.242 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:13 smithi022 ceph-mon[24777]: pgmap v11: 97 pgs: 97 active+clean; 332 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 25 KiB/s rd, 4.0 MiB/s wr, 412 op/s 2024-03-21T12:39:13.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:13 smithi189 ceph-mon[27080]: pgmap v11: 97 pgs: 97 active+clean; 332 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 25 KiB/s rd, 4.0 MiB/s wr, 412 op/s 2024-03-21T12:39:13.847 INFO:teuthology.orchestra.run.smithi022.stdout: 5 134009 12.22 MB/sec execute 133 sec latency 64.460 ms 2024-03-21T12:39:14.847 INFO:teuthology.orchestra.run.smithi022.stdout: 5 134310 12.16 MB/sec execute 134 sec latency 51.547 ms 2024-03-21T12:39:15.308 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:15 smithi022 ceph-mon[24777]: pgmap v12: 97 pgs: 97 active+clean; 340 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 23 KiB/s rd, 6.2 MiB/s wr, 618 op/s 2024-03-21T12:39:15.309 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:15 smithi022 ceph-mon[24777]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' 2024-03-21T12:39:15.309 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:15 smithi022 ceph-mon[24777]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' 2024-03-21T12:39:15.309 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:15 smithi022 ceph-mon[24777]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:39:15.309 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:15 smithi022 ceph-mon[24777]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:39:15.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:15 smithi189 ceph-mon[27080]: pgmap v12: 97 pgs: 97 active+clean; 340 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 23 KiB/s rd, 6.2 MiB/s wr, 618 op/s 2024-03-21T12:39:15.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:15 smithi189 ceph-mon[27080]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' 2024-03-21T12:39:15.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:15 smithi189 ceph-mon[27080]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' 2024-03-21T12:39:15.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:15 smithi189 ceph-mon[27080]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:39:15.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:15 smithi189 ceph-mon[27080]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:39:15.847 INFO:teuthology.orchestra.run.smithi022.stdout: 5 134488 12.09 MB/sec execute 135 sec latency 54.122 ms 2024-03-21T12:39:16.325 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:16 smithi022 ceph-mon[24777]: Updating smithi022:/etc/ceph/ceph.conf 2024-03-21T12:39:16.325 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:16 smithi022 ceph-mon[24777]: Updating smithi189:/etc/ceph/ceph.conf 2024-03-21T12:39:16.325 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:16 smithi022 ceph-mon[24777]: Updating smithi022:/var/lib/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/config/ceph.conf 2024-03-21T12:39:16.325 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:16 smithi022 ceph-mon[24777]: Updating smithi189:/var/lib/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/config/ceph.conf 2024-03-21T12:39:16.325 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:16 smithi022 ceph-mon[24777]: Updating smithi022:/etc/ceph/ceph.client.admin.keyring 2024-03-21T12:39:16.325 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:16 smithi022 ceph-mon[24777]: Updating smithi189:/etc/ceph/ceph.client.admin.keyring 2024-03-21T12:39:16.325 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:16 smithi022 ceph-mon[24777]: Updating smithi022:/var/lib/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/config/ceph.client.admin.keyring 2024-03-21T12:39:16.325 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:16 smithi022 ceph-mon[24777]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' 2024-03-21T12:39:16.325 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:16 smithi022 ceph-mon[24777]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' 2024-03-21T12:39:16.325 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:16 smithi022 ceph-mon[24777]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' 2024-03-21T12:39:16.325 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:16 smithi022 ceph-mon[24777]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' 2024-03-21T12:39:16.326 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:16 smithi022 ceph-mon[24777]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' 2024-03-21T12:39:16.326 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:16 smithi022 ceph-mon[24777]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' 2024-03-21T12:39:16.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:16 smithi189 ceph-mon[27080]: Updating smithi022:/etc/ceph/ceph.conf 2024-03-21T12:39:16.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:16 smithi189 ceph-mon[27080]: Updating smithi189:/etc/ceph/ceph.conf 2024-03-21T12:39:16.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:16 smithi189 ceph-mon[27080]: Updating smithi022:/var/lib/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/config/ceph.conf 2024-03-21T12:39:16.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:16 smithi189 ceph-mon[27080]: Updating smithi189:/var/lib/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/config/ceph.conf 2024-03-21T12:39:16.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:16 smithi189 ceph-mon[27080]: Updating smithi022:/etc/ceph/ceph.client.admin.keyring 2024-03-21T12:39:16.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:16 smithi189 ceph-mon[27080]: Updating smithi189:/etc/ceph/ceph.client.admin.keyring 2024-03-21T12:39:16.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:16 smithi189 ceph-mon[27080]: Updating smithi022:/var/lib/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/config/ceph.client.admin.keyring 2024-03-21T12:39:16.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:16 smithi189 ceph-mon[27080]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' 2024-03-21T12:39:16.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:16 smithi189 ceph-mon[27080]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' 2024-03-21T12:39:16.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:16 smithi189 ceph-mon[27080]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' 2024-03-21T12:39:16.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:16 smithi189 ceph-mon[27080]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' 2024-03-21T12:39:16.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:16 smithi189 ceph-mon[27080]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' 2024-03-21T12:39:16.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:16 smithi189 ceph-mon[27080]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' 2024-03-21T12:39:16.847 INFO:teuthology.orchestra.run.smithi022.stdout: 5 134774 12.08 MB/sec execute 136 sec latency 47.725 ms 2024-03-21T12:39:17.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:17 smithi022 ceph-mon[24777]: pgmap v13: 97 pgs: 97 active+clean; 340 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.2 MiB/s wr, 418 op/s 2024-03-21T12:39:17.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:17 smithi022 ceph-mon[24777]: Updating smithi189:/var/lib/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/config/ceph.client.admin.keyring 2024-03-21T12:39:17.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:17 smithi022 ceph-mon[24777]: pgmap v14: 97 pgs: 97 active+clean; 340 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.9 MiB/s wr, 485 op/s 2024-03-21T12:39:17.345 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:17 smithi022 ceph-mon[24777]: Reconfiguring prometheus.smithi022 (dependencies changed)... 2024-03-21T12:39:17.345 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:17 smithi022 ceph-mon[24777]: Reconfiguring daemon prometheus.smithi022 on smithi022 2024-03-21T12:39:17.345 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:17 smithi022 ceph-mon[24777]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-03-21T12:39:17.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:17 smithi189 ceph-mon[27080]: pgmap v13: 97 pgs: 97 active+clean; 340 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.2 MiB/s wr, 418 op/s 2024-03-21T12:39:17.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:17 smithi189 ceph-mon[27080]: Updating smithi189:/var/lib/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/config/ceph.client.admin.keyring 2024-03-21T12:39:17.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:17 smithi189 ceph-mon[27080]: pgmap v14: 97 pgs: 97 active+clean; 340 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.9 MiB/s wr, 485 op/s 2024-03-21T12:39:17.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:17 smithi189 ceph-mon[27080]: Reconfiguring prometheus.smithi022 (dependencies changed)... 2024-03-21T12:39:17.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:17 smithi189 ceph-mon[27080]: Reconfiguring daemon prometheus.smithi022 on smithi022 2024-03-21T12:39:17.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:17 smithi189 ceph-mon[27080]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-03-21T12:39:17.847 INFO:teuthology.orchestra.run.smithi022.stdout: 5 135190 12.12 MB/sec execute 137 sec latency 51.668 ms 2024-03-21T12:39:18.359 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:18 smithi022 ceph-mon[24777]: pgmap v15: 97 pgs: 97 active+clean; 331 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.3 MiB/s wr, 451 op/s 2024-03-21T12:39:18.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:18 smithi189 ceph-mon[27080]: pgmap v15: 97 pgs: 97 active+clean; 331 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.3 MiB/s wr, 451 op/s 2024-03-21T12:39:18.847 INFO:teuthology.orchestra.run.smithi022.stdout: 5 135704 12.13 MB/sec execute 138 sec latency 47.388 ms 2024-03-21T12:39:19.849 INFO:teuthology.orchestra.run.smithi022.stdout: 5 135983 12.09 MB/sec execute 139 sec latency 74.454 ms 2024-03-21T12:39:20.656 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:20 smithi022 ceph-mon[24777]: pgmap v16: 97 pgs: 97 active+clean; 331 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.3 MiB/s wr, 451 op/s 2024-03-21T12:39:20.847 INFO:teuthology.orchestra.run.smithi022.stdout: 5 136333 12.07 MB/sec execute 140 sec latency 40.835 ms 2024-03-21T12:39:21.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:20 smithi189 ceph-mon[27080]: pgmap v16: 97 pgs: 97 active+clean; 331 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.3 MiB/s wr, 451 op/s 2024-03-21T12:39:21.847 INFO:teuthology.orchestra.run.smithi022.stdout: 5 136653 12.03 MB/sec execute 141 sec latency 41.493 ms 2024-03-21T12:39:22.143 INFO:teuthology.orchestra.run.smithi022.stdout:true 2024-03-21T12:39:22.690 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:22 smithi022 ceph-mon[24777]: pgmap v17: 97 pgs: 97 active+clean; 338 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 7.1 MiB/s wr, 700 op/s 2024-03-21T12:39:22.691 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:22 smithi022 ceph-mon[24777]: from='client.24593 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:39:22.847 INFO:teuthology.orchestra.run.smithi022.stdout: 5 137042 12.05 MB/sec execute 142 sec latency 37.488 ms 2024-03-21T12:39:23.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:22 smithi189 ceph-mon[27080]: pgmap v17: 97 pgs: 97 active+clean; 338 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 7.1 MiB/s wr, 700 op/s 2024-03-21T12:39:23.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:22 smithi189 ceph-mon[27080]: from='client.24593 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:39:23.370 INFO:teuthology.orchestra.run.smithi022.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-21T12:39:23.370 INFO:teuthology.orchestra.run.smithi022.stdout:alertmanager.smithi022 smithi022 *:9093,9094 running (114s) 18s ago 10m 25.7M - 0.23.0 ba2b418f427c 14eb2a3ca318 2024-03-21T12:39:23.370 INFO:teuthology.orchestra.run.smithi022.stdout:crash.smithi022 smithi022 running (10m) 18s ago 10m 6405k - 17.2.0 e1d6a67b021e 47deb70fd250 2024-03-21T12:39:23.370 INFO:teuthology.orchestra.run.smithi022.stdout:crash.smithi189 smithi189 running (10m) 22s ago 10m 6405k - 17.2.0 e1d6a67b021e 9a2e0e9b1809 2024-03-21T12:39:23.370 INFO:teuthology.orchestra.run.smithi022.stdout:grafana.smithi022 smithi022 *:3000 unknown 18s ago 10m - - 2024-03-21T12:39:23.370 INFO:teuthology.orchestra.run.smithi022.stdout:mds.foofs.smithi022.bcnxho smithi022 running (4m) 18s ago 4m 37.0M - 17.2.0 e1d6a67b021e db26b67b8ee7 2024-03-21T12:39:23.370 INFO:teuthology.orchestra.run.smithi022.stdout:mds.foofs.smithi189.lluvko smithi189 running (4m) 22s ago 4m 16.2M - 17.2.0 e1d6a67b021e cf153aff1387 2024-03-21T12:39:23.370 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.smithi022.fektzs smithi022 *:8443,9283,8765 running (49s) 18s ago 11m 479M - 19.0.0-2267-g960fbbe4 19e12293512f d57cebd306b3 2024-03-21T12:39:23.371 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.smithi189.cnxzvu smithi189 *:8443,9283 running (2m) 22s ago 10m 82.9M - 19.0.0-2267-g960fbbe4 19e12293512f 07f3cf369ff7 2024-03-21T12:39:23.371 INFO:teuthology.orchestra.run.smithi022.stdout:mon.smithi022 smithi022 running (12m) 18s ago 12m 57.0M 2048M 17.2.0 e1d6a67b021e 0d248775e0e5 2024-03-21T12:39:23.371 INFO:teuthology.orchestra.run.smithi022.stdout:mon.smithi189 smithi189 running (9m) 22s ago 9m 45.2M 2048M 17.2.0 e1d6a67b021e 82da5f3cd49a 2024-03-21T12:39:23.371 INFO:teuthology.orchestra.run.smithi022.stdout:nfs.foo.0.0.smithi022.ynhbbq smithi022 *:2049 running (4m) 18s ago 4m 256M - 4.0 e1d6a67b021e 33e48e64c60b 2024-03-21T12:39:23.371 INFO:teuthology.orchestra.run.smithi022.stdout:nfs.foo.1.0.smithi189.yzsqml smithi189 *:2049 running (4m) 22s ago 4m 79.5M - 4.0 e1d6a67b021e 48fab2cbdde2 2024-03-21T12:39:23.371 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.smithi022 smithi022 *:9100 running (90s) 18s ago 10m 15.4M - 1.3.1 1dbe0e931976 33287f803f4f 2024-03-21T12:39:23.371 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.smithi189 smithi189 *:9100 running (62s) 22s ago 9m 10.8M - 1.3.1 1dbe0e931976 b51d872e438a 2024-03-21T12:39:23.371 INFO:teuthology.orchestra.run.smithi022.stdout:osd.0 smithi022 running (9m) 18s ago 9m 465M 1937M 17.2.0 e1d6a67b021e 54e6631bd10f 2024-03-21T12:39:23.371 INFO:teuthology.orchestra.run.smithi022.stdout:osd.1 smithi022 running (8m) 18s ago 8m 400M 1937M 17.2.0 e1d6a67b021e 68165c5d09cb 2024-03-21T12:39:23.371 INFO:teuthology.orchestra.run.smithi022.stdout:osd.2 smithi022 running (8m) 18s ago 8m 286M 1937M 17.2.0 e1d6a67b021e fbec9bef8433 2024-03-21T12:39:23.371 INFO:teuthology.orchestra.run.smithi022.stdout:osd.3 smithi022 running (7m) 18s ago 7m 425M 1937M 17.2.0 e1d6a67b021e b1462ffc1b9e 2024-03-21T12:39:23.371 INFO:teuthology.orchestra.run.smithi022.stdout:osd.4 smithi189 running (7m) 22s ago 7m 437M 2704M 17.2.0 e1d6a67b021e 1bedc4eb02ed 2024-03-21T12:39:23.371 INFO:teuthology.orchestra.run.smithi022.stdout:osd.5 smithi189 running (7m) 22s ago 7m 485M 2704M 17.2.0 e1d6a67b021e 25fbab9786a5 2024-03-21T12:39:23.371 INFO:teuthology.orchestra.run.smithi022.stdout:osd.6 smithi189 running (6m) 22s ago 6m 409M 2704M 17.2.0 e1d6a67b021e 0c192fcfd65b 2024-03-21T12:39:23.371 INFO:teuthology.orchestra.run.smithi022.stdout:osd.7 smithi189 running (6m) 22s ago 6m 568M 2704M 17.2.0 e1d6a67b021e 3d0723e1f863 2024-03-21T12:39:23.372 INFO:teuthology.orchestra.run.smithi022.stdout:prometheus.smithi022 smithi022 *:9095 running (65s) 18s ago 9m 53.4M - 2.33.4 514e6a882f6e f632d068e116 2024-03-21T12:39:23.702 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:23 smithi022 ceph-mon[24777]: from='client.14822 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:39:23.847 INFO:teuthology.orchestra.run.smithi022.stdout: 5 137346 12.03 MB/sec execute 143 sec latency 65.864 ms 2024-03-21T12:39:24.022 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-03-21T12:39:24.023 INFO:teuthology.orchestra.run.smithi022.stdout: "mon": { 2024-03-21T12:39:24.023 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-21T12:39:24.023 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:39:24.023 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr": { 2024-03-21T12:39:24.023 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 2 2024-03-21T12:39:24.023 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:39:24.023 INFO:teuthology.orchestra.run.smithi022.stdout: "osd": { 2024-03-21T12:39:24.023 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-03-21T12:39:24.023 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:39:24.023 INFO:teuthology.orchestra.run.smithi022.stdout: "mds": { 2024-03-21T12:39:24.023 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-21T12:39:24.023 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:39:24.023 INFO:teuthology.orchestra.run.smithi022.stdout: "overall": { 2024-03-21T12:39:24.023 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-03-21T12:39:24.024 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 2 2024-03-21T12:39:24.024 INFO:teuthology.orchestra.run.smithi022.stdout: } 2024-03-21T12:39:24.024 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-03-21T12:39:24.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:23 smithi189 ceph-mon[27080]: from='client.14822 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:39:24.574 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-03-21T12:39:24.574 INFO:teuthology.orchestra.run.smithi022.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93", 2024-03-21T12:39:24.574 INFO:teuthology.orchestra.run.smithi022.stdout: "in_progress": true, 2024-03-21T12:39:24.574 INFO:teuthology.orchestra.run.smithi022.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-21T12:39:24.575 INFO:teuthology.orchestra.run.smithi022.stdout: "services_complete": [ 2024-03-21T12:39:24.575 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr" 2024-03-21T12:39:24.575 INFO:teuthology.orchestra.run.smithi022.stdout: ], 2024-03-21T12:39:24.575 INFO:teuthology.orchestra.run.smithi022.stdout: "progress": "2/23 daemons upgraded", 2024-03-21T12:39:24.575 INFO:teuthology.orchestra.run.smithi022.stdout: "message": "", 2024-03-21T12:39:24.575 INFO:teuthology.orchestra.run.smithi022.stdout: "is_paused": false 2024-03-21T12:39:24.575 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-03-21T12:39:24.707 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:24 smithi022 ceph-mon[24777]: from='client.14826 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:39:24.707 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:24 smithi022 ceph-mon[24777]: pgmap v18: 97 pgs: 97 active+clean; 338 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.5 MiB/s wr, 460 op/s 2024-03-21T12:39:24.708 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:24 smithi022 ceph-mon[24777]: from='client.? 172.21.15.22:0/1130191561' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:39:24.847 INFO:teuthology.orchestra.run.smithi022.stdout: 5 137703 11.97 MB/sec execute 144 sec latency 48.340 ms 2024-03-21T12:39:25.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:24 smithi189 ceph-mon[27080]: from='client.14826 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:39:25.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:24 smithi189 ceph-mon[27080]: pgmap v18: 97 pgs: 97 active+clean; 338 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.5 MiB/s wr, 460 op/s 2024-03-21T12:39:25.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:24 smithi189 ceph-mon[27080]: from='client.? 172.21.15.22:0/1130191561' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:39:25.208 INFO:teuthology.orchestra.run.smithi022.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-03-21T12:39:25.208 INFO:teuthology.orchestra.run.smithi022.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-21T12:39:25.208 INFO:teuthology.orchestra.run.smithi022.stdout: daemon grafana.smithi022 on smithi022 is in unknown state 2024-03-21T12:39:25.711 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:25 smithi022 ceph-mon[24777]: from='client.14834 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:39:25.712 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:25 smithi022 ceph-mon[24777]: from='client.? 172.21.15.22:0/1939309057' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-21T12:39:25.712 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:25 smithi022 ceph-mon[24777]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' 2024-03-21T12:39:25.712 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:25 smithi022 ceph-mon[24777]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:39:25.847 INFO:teuthology.orchestra.run.smithi022.stdout: 5 138036 11.92 MB/sec execute 145 sec latency 45.458 ms 2024-03-21T12:39:26.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:25 smithi189 ceph-mon[27080]: from='client.14834 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:39:26.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:25 smithi189 ceph-mon[27080]: from='client.? 172.21.15.22:0/1939309057' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-21T12:39:26.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:25 smithi189 ceph-mon[27080]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' 2024-03-21T12:39:26.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:25 smithi189 ceph-mon[27080]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:39:26.719 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:26 smithi022 ceph-mon[24777]: pgmap v19: 97 pgs: 97 active+clean; 338 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.5 MiB/s wr, 460 op/s 2024-03-21T12:39:26.847 INFO:teuthology.orchestra.run.smithi022.stdout: 5 138518 11.96 MB/sec execute 146 sec latency 37.531 ms 2024-03-21T12:39:26.936 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:26 smithi189 ceph-mon[27080]: pgmap v19: 97 pgs: 97 active+clean; 338 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.5 MiB/s wr, 460 op/s 2024-03-21T12:39:27.850 INFO:teuthology.orchestra.run.smithi022.stdout: 5 139197 12.04 MB/sec execute 147 sec latency 42.814 ms 2024-03-21T12:39:27.974 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:27 smithi022 ceph-mon[24777]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' 2024-03-21T12:39:27.975 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:27 smithi022 ceph-mon[24777]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' 2024-03-21T12:39:27.975 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:27 smithi022 ceph-mon[24777]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-21T12:39:27.975 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:27 smithi022 ceph-mon[24777]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-21T12:39:27.975 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:27 smithi022 ceph-mon[24777]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:39:28.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:27 smithi189 ceph-mon[27080]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' 2024-03-21T12:39:28.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:27 smithi189 ceph-mon[27080]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' 2024-03-21T12:39:28.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:27 smithi189 ceph-mon[27080]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-21T12:39:28.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:27 smithi189 ceph-mon[27080]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-21T12:39:28.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:27 smithi189 ceph-mon[27080]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:39:28.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:28 smithi189 ceph-mon[27080]: pgmap v20: 97 pgs: 97 active+clean; 337 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.0 MiB/s wr, 615 op/s 2024-03-21T12:39:28.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:28 smithi189 ceph-mon[27080]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi189.cnxzvu", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-21T12:39:28.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:28 smithi189 ceph-mon[27080]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-21T12:39:28.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:28 smithi189 ceph-mon[27080]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:39:28.847 INFO:teuthology.orchestra.run.smithi022.stdout: 5 139920 12.06 MB/sec execute 148 sec latency 18.080 ms 2024-03-21T12:39:28.980 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:28 smithi022 ceph-mon[24777]: pgmap v20: 97 pgs: 97 active+clean; 337 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.0 MiB/s wr, 615 op/s 2024-03-21T12:39:28.980 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:28 smithi022 ceph-mon[24777]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi189.cnxzvu", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-21T12:39:28.980 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:28 smithi022 ceph-mon[24777]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-21T12:39:28.981 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:28 smithi022 ceph-mon[24777]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:39:29.847 INFO:teuthology.orchestra.run.smithi022.stdout: 5 140643 12.13 MB/sec execute 149 sec latency 21.139 ms 2024-03-21T12:39:29.989 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:29 smithi022 ceph-mon[24777]: Upgrade: Updating mgr.smithi189.cnxzvu 2024-03-21T12:39:29.989 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:29 smithi022 ceph-mon[24777]: Deploying daemon mgr.smithi189.cnxzvu on smithi189 2024-03-21T12:39:30.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:29 smithi189 ceph-mon[27080]: Upgrade: Updating mgr.smithi189.cnxzvu 2024-03-21T12:39:30.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:29 smithi189 ceph-mon[27080]: Deploying daemon mgr.smithi189.cnxzvu on smithi189 2024-03-21T12:39:30.753 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:30 smithi189 ceph-mon[27080]: pgmap v21: 97 pgs: 97 active+clean; 337 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.4 MiB/s wr, 435 op/s 2024-03-21T12:39:30.847 INFO:teuthology.orchestra.run.smithi022.stdout: 5 141305 12.12 MB/sec execute 150 sec latency 34.976 ms 2024-03-21T12:39:30.996 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:30 smithi022 ceph-mon[24777]: pgmap v21: 97 pgs: 97 active+clean; 337 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.4 MiB/s wr, 435 op/s 2024-03-21T12:39:31.847 INFO:teuthology.orchestra.run.smithi022.stdout: 5 141744 12.11 MB/sec execute 151 sec latency 31.164 ms 2024-03-21T12:39:32.766 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:32 smithi022 ceph-mon[24777]: pgmap v22: 97 pgs: 97 active+clean; 349 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 8.4 MiB/s wr, 814 op/s 2024-03-21T12:39:32.847 INFO:teuthology.orchestra.run.smithi022.stdout: 5 142493 12.22 MB/sec execute 152 sec latency 41.159 ms 2024-03-21T12:39:32.876 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:32 smithi189 ceph-mon[27080]: pgmap v22: 97 pgs: 97 active+clean; 349 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 8.4 MiB/s wr, 814 op/s 2024-03-21T12:39:33.848 INFO:teuthology.orchestra.run.smithi022.stdout: 5 143068 12.23 MB/sec execute 153 sec latency 39.474 ms 2024-03-21T12:39:34.293 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:34 smithi022 ceph-mon[24777]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' 2024-03-21T12:39:34.293 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:34 smithi022 ceph-mon[24777]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' 2024-03-21T12:39:34.293 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:34 smithi022 ceph-mon[24777]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:39:34.294 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:34 smithi022 ceph-mon[24777]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' 2024-03-21T12:39:34.294 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:34 smithi022 ceph-mon[24777]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' 2024-03-21T12:39:34.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:34 smithi189 ceph-mon[27080]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' 2024-03-21T12:39:34.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:34 smithi189 ceph-mon[27080]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' 2024-03-21T12:39:34.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:34 smithi189 ceph-mon[27080]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:39:34.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:34 smithi189 ceph-mon[27080]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' 2024-03-21T12:39:34.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:34 smithi189 ceph-mon[27080]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' 2024-03-21T12:39:34.849 INFO:teuthology.orchestra.run.smithi022.stdout: 5 143504 12.23 MB/sec execute 154 sec latency 43.402 ms 2024-03-21T12:39:35.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:35 smithi189 ceph-mon[27080]: pgmap v23: 97 pgs: 97 active+clean; 349 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 6.0 MiB/s wr, 597 op/s 2024-03-21T12:39:35.304 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:35 smithi022 ceph-mon[24777]: pgmap v23: 97 pgs: 97 active+clean; 349 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 6.0 MiB/s wr, 597 op/s 2024-03-21T12:39:35.849 INFO:teuthology.orchestra.run.smithi022.stdout: 5 143990 12.24 MB/sec execute 155 sec latency 38.315 ms 2024-03-21T12:39:36.850 INFO:teuthology.orchestra.run.smithi022.stdout: 5 144423 12.25 MB/sec execute 156 sec latency 43.016 ms 2024-03-21T12:39:37.325 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:37 smithi022 ceph-mon[24777]: pgmap v24: 97 pgs: 97 active+clean; 349 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 6.0 MiB/s wr, 597 op/s 2024-03-21T12:39:37.407 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:37 smithi189 ceph-mon[27080]: pgmap v24: 97 pgs: 97 active+clean; 349 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 6.0 MiB/s wr, 597 op/s 2024-03-21T12:39:37.850 INFO:teuthology.orchestra.run.smithi022.stdout: 5 144927 12.20 MB/sec execute 157 sec latency 33.316 ms 2024-03-21T12:39:38.850 INFO:teuthology.orchestra.run.smithi022.stdout: 5 145340 12.19 MB/sec execute 158 sec latency 34.456 ms 2024-03-21T12:39:39.034 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:38 smithi022 ceph-mon[24777]: pgmap v25: 97 pgs: 97 active+clean; 347 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 9.6 MiB/s wr, 915 op/s 2024-03-21T12:39:39.034 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:38 smithi022 ceph-mon[24777]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' 2024-03-21T12:39:39.034 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:38 smithi022 ceph-mon[24777]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' 2024-03-21T12:39:39.244 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:38 smithi189 ceph-mon[27080]: pgmap v25: 97 pgs: 97 active+clean; 347 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 9.6 MiB/s wr, 915 op/s 2024-03-21T12:39:39.244 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:38 smithi189 ceph-mon[27080]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' 2024-03-21T12:39:39.244 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:38 smithi189 ceph-mon[27080]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' 2024-03-21T12:39:39.850 INFO:teuthology.orchestra.run.smithi022.stdout: 5 145971 12.26 MB/sec execute 159 sec latency 80.002 ms 2024-03-21T12:39:40.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:40 smithi189 ceph-mon[27080]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' 2024-03-21T12:39:40.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:40 smithi189 ceph-mon[27080]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' 2024-03-21T12:39:40.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:40 smithi189 ceph-mon[27080]: pgmap v26: 97 pgs: 97 active+clean; 347 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 7.6 MiB/s wr, 697 op/s 2024-03-21T12:39:40.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:40 smithi189 ceph-mon[27080]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:39:40.543 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:40 smithi022 ceph-mon[24777]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' 2024-03-21T12:39:40.544 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:40 smithi022 ceph-mon[24777]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' 2024-03-21T12:39:40.544 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:40 smithi022 ceph-mon[24777]: pgmap v26: 97 pgs: 97 active+clean; 347 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 7.6 MiB/s wr, 697 op/s 2024-03-21T12:39:40.544 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:40 smithi022 ceph-mon[24777]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:39:40.850 INFO:teuthology.orchestra.run.smithi022.stdout: 5 146551 12.29 MB/sec execute 160 sec latency 30.443 ms 2024-03-21T12:39:41.850 INFO:teuthology.orchestra.run.smithi022.stdout: 5 146774 12.23 MB/sec execute 161 sec latency 41.793 ms 2024-03-21T12:39:42.834 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:42 smithi022 ceph-mon[24777]: pgmap v27: 97 pgs: 97 active+clean; 342 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 14 KiB/s rd, 10 MiB/s wr, 989 op/s 2024-03-21T12:39:42.835 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:42 smithi022 ceph-mon[24777]: Standby manager daemon smithi189.cnxzvu restarted 2024-03-21T12:39:42.835 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:42 smithi022 ceph-mon[24777]: Standby manager daemon smithi189.cnxzvu started 2024-03-21T12:39:42.835 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:42 smithi022 ceph-mon[24777]: from='mgr.? 172.21.15.189:0/3568493208' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi189.cnxzvu/crt"}]: dispatch 2024-03-21T12:39:42.835 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:42 smithi022 ceph-mon[24777]: from='mgr.? 172.21.15.189:0/3568493208' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-21T12:39:42.835 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:42 smithi022 ceph-mon[24777]: from='mgr.? 172.21.15.189:0/3568493208' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi189.cnxzvu/key"}]: dispatch 2024-03-21T12:39:42.835 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:42 smithi022 ceph-mon[24777]: from='mgr.? 172.21.15.189:0/3568493208' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-21T12:39:42.850 INFO:teuthology.orchestra.run.smithi022.stdout: 5 147185 12.23 MB/sec execute 162 sec latency 48.247 ms 2024-03-21T12:39:43.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:42 smithi189 ceph-mon[27080]: pgmap v27: 97 pgs: 97 active+clean; 342 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 14 KiB/s rd, 10 MiB/s wr, 989 op/s 2024-03-21T12:39:43.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:42 smithi189 ceph-mon[27080]: Standby manager daemon smithi189.cnxzvu restarted 2024-03-21T12:39:43.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:42 smithi189 ceph-mon[27080]: Standby manager daemon smithi189.cnxzvu started 2024-03-21T12:39:43.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:42 smithi189 ceph-mon[27080]: from='mgr.? 172.21.15.189:0/3568493208' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi189.cnxzvu/crt"}]: dispatch 2024-03-21T12:39:43.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:42 smithi189 ceph-mon[27080]: from='mgr.? 172.21.15.189:0/3568493208' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-21T12:39:43.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:42 smithi189 ceph-mon[27080]: from='mgr.? 172.21.15.189:0/3568493208' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi189.cnxzvu/key"}]: dispatch 2024-03-21T12:39:43.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:42 smithi189 ceph-mon[27080]: from='mgr.? 172.21.15.189:0/3568493208' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-21T12:39:43.850 INFO:teuthology.orchestra.run.smithi022.stdout: 5 147562 12.23 MB/sec execute 163 sec latency 41.349 ms 2024-03-21T12:39:43.872 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:43 smithi022 ceph-mon[24777]: mgrmap e34: smithi022.fektzs(active, since 48s), standbys: smithi189.cnxzvu 2024-03-21T12:39:44.020 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:43 smithi189 ceph-mon[27080]: mgrmap e34: smithi022.fektzs(active, since 48s), standbys: smithi189.cnxzvu 2024-03-21T12:39:44.850 INFO:teuthology.orchestra.run.smithi022.stdout: 5 148038 12.24 MB/sec execute 164 sec latency 52.010 ms 2024-03-21T12:39:45.142 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:45 smithi022 ceph-mon[24777]: pgmap v28: 97 pgs: 97 active+clean; 342 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 6.5 MiB/s wr, 610 op/s 2024-03-21T12:39:45.142 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:45 smithi022 ceph-mon[24777]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' 2024-03-21T12:39:45.142 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:45 smithi022 ceph-mon[24777]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' 2024-03-21T12:39:45.142 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:45 smithi022 ceph-mon[24777]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' 2024-03-21T12:39:45.142 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:45 smithi022 ceph-mon[24777]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' 2024-03-21T12:39:45.142 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:45 smithi022 ceph-mon[24777]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:39:45.142 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:45 smithi022 ceph-mon[24777]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:39:45.142 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:45 smithi022 ceph-mon[24777]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' 2024-03-21T12:39:45.142 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:45 smithi022 ceph-mon[24777]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' 2024-03-21T12:39:45.143 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:45 smithi022 ceph-mon[24777]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:39:45.143 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:45 smithi022 ceph-mon[24777]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:39:45.143 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:45 smithi022 ceph-mon[24777]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' 2024-03-21T12:39:45.143 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:45 smithi022 ceph-mon[24777]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi022.fektzs"}]: dispatch 2024-03-21T12:39:45.143 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:45 smithi022 ceph-mon[24777]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi022.fektzs"}]': finished 2024-03-21T12:39:45.143 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:45 smithi022 ceph-mon[24777]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi189.cnxzvu"}]: dispatch 2024-03-21T12:39:45.143 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:45 smithi022 ceph-mon[24777]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi189.cnxzvu"}]': finished 2024-03-21T12:39:45.143 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:45 smithi022 ceph-mon[24777]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-03-21T12:39:45.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:45 smithi189 ceph-mon[27080]: pgmap v28: 97 pgs: 97 active+clean; 342 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 6.5 MiB/s wr, 610 op/s 2024-03-21T12:39:45.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:45 smithi189 ceph-mon[27080]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' 2024-03-21T12:39:45.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:45 smithi189 ceph-mon[27080]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' 2024-03-21T12:39:45.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:45 smithi189 ceph-mon[27080]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' 2024-03-21T12:39:45.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:45 smithi189 ceph-mon[27080]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' 2024-03-21T12:39:45.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:45 smithi189 ceph-mon[27080]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:39:45.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:45 smithi189 ceph-mon[27080]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:39:45.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:45 smithi189 ceph-mon[27080]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' 2024-03-21T12:39:45.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:45 smithi189 ceph-mon[27080]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' 2024-03-21T12:39:45.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:45 smithi189 ceph-mon[27080]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:39:45.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:45 smithi189 ceph-mon[27080]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:39:45.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:45 smithi189 ceph-mon[27080]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' 2024-03-21T12:39:45.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:45 smithi189 ceph-mon[27080]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi022.fektzs"}]: dispatch 2024-03-21T12:39:45.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:45 smithi189 ceph-mon[27080]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi022.fektzs"}]': finished 2024-03-21T12:39:45.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:45 smithi189 ceph-mon[27080]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi189.cnxzvu"}]: dispatch 2024-03-21T12:39:45.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:45 smithi189 ceph-mon[27080]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi189.cnxzvu"}]': finished 2024-03-21T12:39:45.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:45 smithi189 ceph-mon[27080]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-03-21T12:39:45.850 INFO:teuthology.orchestra.run.smithi022.stdout: 5 148520 12.19 MB/sec execute 165 sec latency 55.258 ms 2024-03-21T12:39:46.153 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:46 smithi022 ceph-mon[24777]: pgmap v29: 97 pgs: 97 active+clean; 342 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 7.0 MiB/s wr, 658 op/s 2024-03-21T12:39:46.154 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:46 smithi022 ceph-mon[24777]: Upgrade: Setting container_image for all mgr 2024-03-21T12:39:46.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:46 smithi189 ceph-mon[27080]: pgmap v29: 97 pgs: 97 active+clean; 342 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 7.0 MiB/s wr, 658 op/s 2024-03-21T12:39:46.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:46 smithi189 ceph-mon[27080]: Upgrade: Setting container_image for all mgr 2024-03-21T12:39:46.850 INFO:teuthology.orchestra.run.smithi022.stdout: 5 148961 12.20 MB/sec execute 166 sec latency 37.589 ms 2024-03-21T12:39:47.850 INFO:teuthology.orchestra.run.smithi022.stdout: 5 149441 12.24 MB/sec execute 167 sec latency 47.582 ms 2024-03-21T12:39:48.189 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:48 smithi022 ceph-mon[24777]: pgmap v30: 97 pgs: 97 active+clean; 336 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 14 KiB/s rd, 9.6 MiB/s wr, 927 op/s 2024-03-21T12:39:48.190 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:48 smithi022 ceph-mon[24777]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' 2024-03-21T12:39:48.190 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:48 smithi022 ceph-mon[24777]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-21T12:39:48.190 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:48 smithi022 ceph-mon[24777]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-21T12:39:48.190 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:48 smithi022 ceph-mon[24777]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:39:48.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:48 smithi189 ceph-mon[27080]: pgmap v30: 97 pgs: 97 active+clean; 336 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 14 KiB/s rd, 9.6 MiB/s wr, 927 op/s 2024-03-21T12:39:48.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:48 smithi189 ceph-mon[27080]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' 2024-03-21T12:39:48.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:48 smithi189 ceph-mon[27080]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-21T12:39:48.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:48 smithi189 ceph-mon[27080]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-21T12:39:48.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:48 smithi189 ceph-mon[27080]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:39:48.850 INFO:teuthology.orchestra.run.smithi022.stdout: 5 149960 12.25 MB/sec execute 168 sec latency 62.937 ms 2024-03-21T12:39:49.214 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:49 smithi022 ceph-mon[24777]: Upgrade: Updating mon.smithi022 2024-03-21T12:39:49.214 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:49 smithi022 ceph-mon[24777]: Deploying daemon mon.smithi022 on smithi022 2024-03-21T12:39:49.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:49 smithi189 ceph-mon[27080]: Upgrade: Updating mon.smithi022 2024-03-21T12:39:49.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:49 smithi189 ceph-mon[27080]: Deploying daemon mon.smithi022 on smithi022 2024-03-21T12:39:49.850 INFO:teuthology.orchestra.run.smithi022.stdout: 5 150188 12.21 MB/sec execute 169 sec latency 57.394 ms 2024-03-21T12:39:50.252 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:50 smithi022 ceph-mon[24777]: pgmap v31: 97 pgs: 97 active+clean; 336 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 5.7 MiB/s wr, 584 op/s 2024-03-21T12:39:50.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:50 smithi189 ceph-mon[27080]: pgmap v31: 97 pgs: 97 active+clean; 336 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 5.7 MiB/s wr, 584 op/s 2024-03-21T12:39:50.850 INFO:teuthology.orchestra.run.smithi022.stdout: 5 150543 12.20 MB/sec execute 170 sec latency 38.754 ms 2024-03-21T12:39:51.263 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:51 smithi022 systemd[1]: Stopping Ceph mon.smithi022 for 3fc7f0f2-e77e-11ee-95cd-87774f69a715... 2024-03-21T12:39:51.514 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:51 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-mon-smithi022[24773]: 2024-03-21T12:39:51.503+0000 7fd45dd16700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi022 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-03-21T12:39:51.514 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:51 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-mon-smithi022[24773]: 2024-03-21T12:39:51.503+0000 7fd45dd16700 -1 mon.smithi022@0(leader) e2 *** Got Signal Terminated *** 2024-03-21T12:39:51.759 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:51 smithi022 podman[66884]: 2024-03-21 12:39:51.526032986 +0000 UTC m=+0.152922852 container died 0d248775e0e517f2b9eb507a302cf35719effa905a33af734b742af906a3edff (image=quay.io/ceph/ceph:v17.2.0, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-mon-smithi022, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, vcs-type=git, ceph=True, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, distribution-scope=public, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., RELEASE=HEAD, io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-03-21T12:39:51.856 INFO:teuthology.orchestra.run.smithi022.stdout: 5 150886 12.16 MB/sec execute 171 sec latency 51.768 ms 2024-03-21T12:39:52.274 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:52 smithi022 podman[66884]: 2024-03-21 12:39:52.225914135 +0000 UTC m=+0.852804005 container cleanup 0d248775e0e517f2b9eb507a302cf35719effa905a33af734b742af906a3edff (image=quay.io/ceph/ceph:v17.2.0, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-mon-smithi022, build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, com.redhat.component=centos-stream-container) 2024-03-21T12:39:52.274 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:52 smithi022 bash[66884]: ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-mon-smithi022 2024-03-21T12:39:52.526 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:52 smithi022 podman[66902]: 2024-03-21 12:39:52.5108306 +0000 UTC m=+0.968807805 container remove 0d248775e0e517f2b9eb507a302cf35719effa905a33af734b742af906a3edff (image=quay.io/ceph/ceph:v17.2.0, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-mon-smithi022, ceph=True, version=8, GIT_CLEAN=True, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, vcs-type=git, vendor=Red Hat, Inc., release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, distribution-scope=public, io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=) 2024-03-21T12:39:52.778 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:52 smithi022 systemd[1]: ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715@mon.smithi022.service: Deactivated successfully. 2024-03-21T12:39:52.778 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:52 smithi022 systemd[1]: Stopped Ceph mon.smithi022 for 3fc7f0f2-e77e-11ee-95cd-87774f69a715. 2024-03-21T12:39:52.778 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:52 smithi022 systemd[1]: ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715@mon.smithi022.service: Consumed 16.806s CPU time. 2024-03-21T12:39:52.856 INFO:teuthology.orchestra.run.smithi022.stdout: 5 151181 12.15 MB/sec execute 172 sec latency 60.014 ms 2024-03-21T12:39:53.043 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:52 smithi022 systemd[1]: Starting Ceph mon.smithi022 for 3fc7f0f2-e77e-11ee-95cd-87774f69a715... 2024-03-21T12:39:53.298 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 podman[67028]: 2024-03-21 12:39:53.147957877 +0000 UTC m=+0.022194239 image pull 19e12293512ff7978c8adb5f72a829060dd47f07bfd6903b6b3178d2ebc29e4e quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93 2024-03-21T12:39:53.298 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 podman[67028]: 2024-03-21 12:39:53.270876483 +0000 UTC m=+0.145112831 container create c620c9071b2c5e4c45ef86ebd2c1e7f031673bb2ec8844d2cdb9d9b86d552e78 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-mon-smithi022, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240319, CEPH_POINT_RELEASE=, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe) 2024-03-21T12:39:53.801 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 podman[67028]: 2024-03-21 12:39:53.635451895 +0000 UTC m=+0.509688247 container init c620c9071b2c5e4c45ef86ebd2c1e7f031673bb2ec8844d2cdb9d9b86d552e78 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-mon-smithi022, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.build-date=20240319, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD) 2024-03-21T12:39:53.801 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 podman[67028]: 2024-03-21 12:39:53.643351851 +0000 UTC m=+0.517588204 container start c620c9071b2c5e4c45ef86ebd2c1e7f031673bb2ec8844d2cdb9d9b86d552e78 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-mon-smithi022, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240319, maintainer=Guillaume Abrioux , GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_CLEAN=True, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, GIT_BRANCH=HEAD) 2024-03-21T12:39:53.801 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: set uid:gid to 167:167 (ceph:ceph) 2024-03-21T12:39:53.801 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev), process ceph-mon, pid 2 2024-03-21T12:39:53.801 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: pidfile_write: ignore empty --pid-file 2024-03-21T12:39:53.801 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: load: jerasure load: lrc 2024-03-21T12:39:53.801 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: RocksDB version: 7.9.2 2024-03-21T12:39:53.801 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Git sha 0 2024-03-21T12:39:53.801 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Compile date 2024-03-21 08:53:26 2024-03-21T12:39:53.802 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: DB SUMMARY 2024-03-21T12:39:53.802 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: DB Session ID: BF0840UWVU2R7ZJYWC4M 2024-03-21T12:39:53.802 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: CURRENT file: CURRENT 2024-03-21T12:39:53.802 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: IDENTITY file: IDENTITY 2024-03-21T12:39:53.802 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: MANIFEST file: MANIFEST-000015 size: 978 Bytes 2024-03-21T12:39:53.802 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi022/store.db dir, Total Num: 1, files: 000030.sst 2024-03-21T12:39:53.802 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi022/store.db: 000028.log size: 2672221 ; 2024-03-21T12:39:53.802 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.error_if_exists: 0 2024-03-21T12:39:53.803 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.create_if_missing: 0 2024-03-21T12:39:53.803 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.paranoid_checks: 1 2024-03-21T12:39:53.803 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.flush_verify_memtable_count: 1 2024-03-21T12:39:53.803 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-03-21T12:39:53.803 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-03-21T12:39:53.803 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.env: 0x55be1c128b20 2024-03-21T12:39:53.803 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.fs: PosixFileSystem 2024-03-21T12:39:53.803 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.info_log: 0x55be1d2ed7c0 2024-03-21T12:39:53.803 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.max_file_opening_threads: 16 2024-03-21T12:39:53.804 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.statistics: (nil) 2024-03-21T12:39:53.804 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.use_fsync: 0 2024-03-21T12:39:53.804 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.max_log_file_size: 0 2024-03-21T12:39:53.804 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-03-21T12:39:53.805 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.log_file_time_to_roll: 0 2024-03-21T12:39:53.805 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.keep_log_file_num: 1000 2024-03-21T12:39:53.805 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.recycle_log_file_num: 0 2024-03-21T12:39:53.805 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.allow_fallocate: 1 2024-03-21T12:39:53.805 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.allow_mmap_reads: 0 2024-03-21T12:39:53.805 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.allow_mmap_writes: 0 2024-03-21T12:39:53.805 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.use_direct_reads: 0 2024-03-21T12:39:53.805 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-03-21T12:39:53.805 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.create_missing_column_families: 0 2024-03-21T12:39:53.805 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.db_log_dir: 2024-03-21T12:39:53.805 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.wal_dir: 2024-03-21T12:39:53.805 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.table_cache_numshardbits: 6 2024-03-21T12:39:53.805 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.WAL_ttl_seconds: 0 2024-03-21T12:39:53.805 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.WAL_size_limit_MB: 0 2024-03-21T12:39:53.805 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-03-21T12:39:53.805 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-03-21T12:39:53.806 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.is_fd_close_on_exec: 1 2024-03-21T12:39:53.806 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.advise_random_on_open: 1 2024-03-21T12:39:53.806 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.db_write_buffer_size: 0 2024-03-21T12:39:53.806 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.write_buffer_manager: 0x55be1d2be1e0 2024-03-21T12:39:53.806 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-03-21T12:39:53.806 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-03-21T12:39:53.806 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.use_adaptive_mutex: 0 2024-03-21T12:39:53.806 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.rate_limiter: (nil) 2024-03-21T12:39:53.806 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-03-21T12:39:53.806 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.wal_recovery_mode: 2 2024-03-21T12:39:53.806 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.enable_thread_tracking: 0 2024-03-21T12:39:53.806 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.enable_pipelined_write: 0 2024-03-21T12:39:53.806 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.unordered_write: 0 2024-03-21T12:39:53.806 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-03-21T12:39:53.807 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-03-21T12:39:53.807 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-03-21T12:39:53.807 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-03-21T12:39:53.807 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.row_cache: None 2024-03-21T12:39:53.807 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.wal_filter: None 2024-03-21T12:39:53.807 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-03-21T12:39:53.807 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.allow_ingest_behind: 0 2024-03-21T12:39:53.807 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.two_write_queues: 0 2024-03-21T12:39:53.807 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.manual_wal_flush: 0 2024-03-21T12:39:53.807 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.wal_compression: 0 2024-03-21T12:39:53.807 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.atomic_flush: 0 2024-03-21T12:39:53.808 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-03-21T12:39:53.808 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.persist_stats_to_disk: 0 2024-03-21T12:39:53.808 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.write_dbid_to_manifest: 0 2024-03-21T12:39:53.808 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.log_readahead_size: 0 2024-03-21T12:39:53.808 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-03-21T12:39:53.808 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.best_efforts_recovery: 0 2024-03-21T12:39:53.808 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-03-21T12:39:53.808 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-03-21T12:39:53.808 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.allow_data_in_errors: 0 2024-03-21T12:39:53.808 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.db_host_id: __hostname__ 2024-03-21T12:39:53.808 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.enforce_single_del_contracts: true 2024-03-21T12:39:53.808 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.max_background_jobs: 2 2024-03-21T12:39:53.808 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.max_background_compactions: -1 2024-03-21T12:39:53.808 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.max_subcompactions: 1 2024-03-21T12:39:53.808 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-03-21T12:39:53.808 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-03-21T12:39:53.809 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.delayed_write_rate : 16777216 2024-03-21T12:39:53.809 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.max_total_wal_size: 0 2024-03-21T12:39:53.809 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-03-21T12:39:53.809 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.stats_dump_period_sec: 600 2024-03-21T12:39:53.809 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.stats_persist_period_sec: 600 2024-03-21T12:39:53.811 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-03-21T12:39:53.811 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.max_open_files: -1 2024-03-21T12:39:53.811 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.bytes_per_sync: 0 2024-03-21T12:39:53.811 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.wal_bytes_per_sync: 0 2024-03-21T12:39:53.811 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.strict_bytes_per_sync: 0 2024-03-21T12:39:53.811 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.compaction_readahead_size: 0 2024-03-21T12:39:53.811 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.max_background_flushes: -1 2024-03-21T12:39:53.811 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Compression algorithms supported: 2024-03-21T12:39:53.812 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-03-21T12:39:53.812 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: kZSTD supported: 0 2024-03-21T12:39:53.812 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: kXpressCompression supported: 0 2024-03-21T12:39:53.812 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: kLZ4HCCompression supported: 1 2024-03-21T12:39:53.812 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: kZlibCompression supported: 1 2024-03-21T12:39:53.812 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: kSnappyCompression supported: 1 2024-03-21T12:39:53.812 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: kLZ4Compression supported: 1 2024-03-21T12:39:53.812 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: kBZip2Compression supported: 0 2024-03-21T12:39:53.812 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Fast CRC32 supported: Supported on x86 2024-03-21T12:39:53.812 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: DMutex implementation: pthread_mutex_t 2024-03-21T12:39:53.812 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi022/store.db/MANIFEST-000015 2024-03-21T12:39:53.813 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-03-21T12:39:53.813 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-03-21T12:39:53.813 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.merge_operator: 2024-03-21T12:39:53.813 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.compaction_filter: None 2024-03-21T12:39:53.813 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.compaction_filter_factory: None 2024-03-21T12:39:53.813 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.sst_partitioner_factory: None 2024-03-21T12:39:53.813 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.memtable_factory: SkipListFactory 2024-03-21T12:39:53.813 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.table_factory: BlockBasedTable 2024-03-21T12:39:53.813 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55be1d2ed8e0) 2024-03-21T12:39:53.813 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: cache_index_and_filter_blocks: 1 2024-03-21T12:39:53.813 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-03-21T12:39:53.813 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-03-21T12:39:53.814 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: pin_top_level_index_and_filter: 1 2024-03-21T12:39:53.814 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: index_type: 0 2024-03-21T12:39:53.814 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: data_block_index_type: 0 2024-03-21T12:39:53.814 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: index_shortening: 1 2024-03-21T12:39:53.814 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: data_block_hash_table_util_ratio: 0.750000 2024-03-21T12:39:53.814 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: checksum: 4 2024-03-21T12:39:53.814 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: no_block_cache: 0 2024-03-21T12:39:53.814 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: block_cache: 0x55be1d2f1090 2024-03-21T12:39:53.814 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: block_cache_name: BinnedLRUCache 2024-03-21T12:39:53.814 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: block_cache_options: 2024-03-21T12:39:53.814 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: capacity : 536870912 2024-03-21T12:39:53.814 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: num_shard_bits : 4 2024-03-21T12:39:53.814 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: strict_capacity_limit : 0 2024-03-21T12:39:53.814 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: high_pri_pool_ratio: 0.000 2024-03-21T12:39:53.815 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: block_cache_compressed: (nil) 2024-03-21T12:39:53.815 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: persistent_cache: (nil) 2024-03-21T12:39:53.815 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: block_size: 4096 2024-03-21T12:39:53.815 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: block_size_deviation: 10 2024-03-21T12:39:53.815 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: block_restart_interval: 16 2024-03-21T12:39:53.815 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: index_block_restart_interval: 1 2024-03-21T12:39:53.815 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: metadata_block_size: 4096 2024-03-21T12:39:53.815 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: partition_filters: 0 2024-03-21T12:39:53.815 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: use_delta_encoding: 1 2024-03-21T12:39:53.815 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: filter_policy: bloomfilter 2024-03-21T12:39:53.815 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: whole_key_filtering: 1 2024-03-21T12:39:53.815 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: verify_compression: 0 2024-03-21T12:39:53.815 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: read_amp_bytes_per_bit: 0 2024-03-21T12:39:53.817 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: format_version: 5 2024-03-21T12:39:53.817 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: enable_index_compression: 1 2024-03-21T12:39:53.818 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: block_align: 0 2024-03-21T12:39:53.818 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: max_auto_readahead_size: 262144 2024-03-21T12:39:53.818 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: prepopulate_block_cache: 0 2024-03-21T12:39:53.818 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: initial_auto_readahead_size: 8192 2024-03-21T12:39:53.818 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: num_file_reads_for_auto_readahead: 2 2024-03-21T12:39:53.818 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.write_buffer_size: 33554432 2024-03-21T12:39:53.818 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.max_write_buffer_number: 2 2024-03-21T12:39:53.818 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.compression: NoCompression 2024-03-21T12:39:53.818 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.bottommost_compression: Disabled 2024-03-21T12:39:53.818 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.prefix_extractor: nullptr 2024-03-21T12:39:53.818 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-03-21T12:39:53.818 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.num_levels: 7 2024-03-21T12:39:53.818 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-03-21T12:39:53.818 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-03-21T12:39:53.818 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-03-21T12:39:53.819 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-03-21T12:39:53.819 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-03-21T12:39:53.819 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-03-21T12:39:53.819 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-03-21T12:39:53.819 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-03-21T12:39:53.819 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-03-21T12:39:53.819 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-03-21T12:39:53.819 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-03-21T12:39:53.819 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-03-21T12:39:53.819 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.compression_opts.window_bits: -14 2024-03-21T12:39:53.819 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.compression_opts.level: 32767 2024-03-21T12:39:53.819 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.compression_opts.strategy: 0 2024-03-21T12:39:53.819 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-03-21T12:39:53.819 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-03-21T12:39:53.819 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-03-21T12:39:53.819 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-03-21T12:39:53.820 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.compression_opts.enabled: false 2024-03-21T12:39:53.821 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-03-21T12:39:53.822 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-03-21T12:39:53.822 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-03-21T12:39:53.822 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-03-21T12:39:53.822 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.target_file_size_base: 67108864 2024-03-21T12:39:53.822 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.target_file_size_multiplier: 1 2024-03-21T12:39:53.822 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-03-21T12:39:53.822 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-03-21T12:39:53.822 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-03-21T12:39:53.822 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-03-21T12:39:53.822 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-03-21T12:39:53.822 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-03-21T12:39:53.822 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-03-21T12:39:53.823 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-03-21T12:39:53.823 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-03-21T12:39:53.823 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-03-21T12:39:53.823 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-03-21T12:39:53.823 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-03-21T12:39:53.823 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-03-21T12:39:53.823 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.arena_block_size: 1048576 2024-03-21T12:39:53.823 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-03-21T12:39:53.823 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-03-21T12:39:53.824 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.disable_auto_compactions: 0 2024-03-21T12:39:53.824 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-03-21T12:39:53.824 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-03-21T12:39:53.824 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-03-21T12:39:53.824 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-03-21T12:39:53.824 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-03-21T12:39:53.824 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-03-21T12:39:53.824 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-03-21T12:39:53.824 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-03-21T12:39:53.824 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-03-21T12:39:53.824 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-03-21T12:39:53.824 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.table_properties_collectors: 2024-03-21T12:39:53.824 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.inplace_update_support: 0 2024-03-21T12:39:53.824 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.inplace_update_num_locks: 10000 2024-03-21T12:39:53.824 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-03-21T12:39:53.825 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-03-21T12:39:53.825 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.memtable_huge_page_size: 0 2024-03-21T12:39:53.825 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.bloom_locality: 0 2024-03-21T12:39:53.825 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.max_successive_merges: 0 2024-03-21T12:39:53.825 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.optimize_filters_for_hits: 0 2024-03-21T12:39:53.825 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.paranoid_file_checks: 0 2024-03-21T12:39:53.825 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.force_consistency_checks: 1 2024-03-21T12:39:53.825 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.report_bg_io_stats: 0 2024-03-21T12:39:53.825 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.ttl: 2592000 2024-03-21T12:39:53.825 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.periodic_compaction_seconds: 0 2024-03-21T12:39:53.825 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-03-21T12:39:53.825 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-03-21T12:39:53.825 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.enable_blob_files: false 2024-03-21T12:39:53.825 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.min_blob_size: 0 2024-03-21T12:39:53.825 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.blob_file_size: 268435456 2024-03-21T12:39:53.826 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.blob_compression_type: NoCompression 2024-03-21T12:39:53.826 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.enable_blob_garbage_collection: false 2024-03-21T12:39:53.826 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-03-21T12:39:53.826 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-03-21T12:39:53.826 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-03-21T12:39:53.826 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.blob_file_starting_level: 0 2024-03-21T12:39:53.826 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-03-21T12:39:53.826 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 30.sst 2024-03-21T12:39:53.826 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-03-21T12:39:53.826 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi022/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 32, last_sequence is 11813, log_number is 28,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-03-21T12:39:53.826 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 28 2024-03-21T12:39:53.826 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: e89af291-bcac-42d2-8583-b4ba1c2dea86 2024-03-21T12:39:53.826 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: EVENT_LOG_v1 {"time_micros": 1711024793681150, "job": 1, "event": "recovery_started", "wal_files": [28]} 2024-03-21T12:39:53.826 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #28 mode 2 2024-03-21T12:39:53.826 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: EVENT_LOG_v1 {"time_micros": 1711024793691256, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 33, "file_size": 2485681, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 11807, "largest_seqno": 12901, "table_properties": {"data_size": 2480189, "index_size": 3257, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1221, "raw_key_size": 10839, "raw_average_key_size": 23, "raw_value_size": 2469942, "raw_average_value_size": 5266, "num_data_blocks": 148, "num_entries": 469, "num_filter_entries": 469, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1711024793, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "e89af291-bcac-42d2-8583-b4ba1c2dea86", "db_session_id": "BF0840UWVU2R7ZJYWC4M", "orig_file_number": 33, "seqno_to_time_mapping": "N/A"}} 2024-03-21T12:39:53.827 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: EVENT_LOG_v1 {"time_micros": 1711024793691347, "job": 1, "event": "recovery_finished"} 2024-03-21T12:39:53.827 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: [db/version_set.cc:5047] Creating manifest 35 2024-03-21T12:39:53.827 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-03-21T12:39:53.827 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-smithi022/store.db/000028.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-03-21T12:39:53.827 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55be1d3de000 2024-03-21T12:39:53.827 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: DB pointer 0x55be1d3ca000 2024-03-21T12:39:53.827 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-03-21T12:39:53.827 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-03-21T12:39:53.827 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: ** DB Stats ** 2024-03-21T12:39:53.827 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-03-21T12:39:53.827 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-03-21T12:39:53.828 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-03-21T12:39:53.828 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-03-21T12:39:53.828 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-03-21T12:39:53.828 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-03-21T12:39:53.828 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-03-21T12:39:53.828 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: 2024-03-21T12:39:53.828 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: ** Compaction Stats [default] ** 2024-03-21T12:39:53.828 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-03-21T12:39:53.828 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-03-21T12:39:53.828 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: L0 1/0 2.37 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 449.4 0.01 0.00 1 0.005 0 0 0.0 0.0 2024-03-21T12:39:53.829 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: L6 1/0 6.53 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-03-21T12:39:53.829 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: Sum 2/0 8.90 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 449.4 0.01 0.00 1 0.005 0 0 0.0 0.0 2024-03-21T12:39:53.829 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 449.4 0.01 0.00 1 0.005 0 0 0.0 0.0 2024-03-21T12:39:53.829 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: 2024-03-21T12:39:53.829 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: ** Compaction Stats [default] ** 2024-03-21T12:39:53.829 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-03-21T12:39:53.829 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-03-21T12:39:53.829 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 449.4 0.01 0.00 1 0.005 0 0 0.0 0.0 2024-03-21T12:39:53.829 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: 2024-03-21T12:39:53.830 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-03-21T12:39:53.830 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: 2024-03-21T12:39:53.830 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-03-21T12:39:53.830 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: Flush(GB): cumulative 0.002, interval 0.002 2024-03-21T12:39:53.830 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-03-21T12:39:53.830 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-03-21T12:39:53.830 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-03-21T12:39:53.830 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: AddFile(Keys): cumulative 0, interval 0 2024-03-21T12:39:53.831 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: Cumulative compaction: 0.00 GB write, 183.93 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-21T12:39:53.831 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: Interval compaction: 0.00 GB write, 183.93 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-21T12:39:53.831 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-03-21T12:39:53.832 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: Block cache BinnedLRUCache@0x55be1d2f1090#2 capacity: 512.00 MB usage: 4.61 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.2e-05 secs_since: 0 2024-03-21T12:39:53.832 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.28 KB,0.000244379%) IndexBlock(1,3.33 KB,0.000634789%) Misc(1,0.00 KB,0%) 2024-03-21T12:39:53.832 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: 2024-03-21T12:39:53.832 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: ** File Read Latency Histogram By Level [default] ** 2024-03-21T12:39:53.832 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: starting mon.smithi022 rank 0 at public addrs [v2:172.21.15.22:3300/0,v1:172.21.15.22:6789/0] at bind addrs [v2:172.21.15.22:3300/0,v1:172.21.15.22:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi022 fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 2024-03-21T12:39:53.832 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: mon.smithi022@-1(???) e2 preinit fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 2024-03-21T12:39:53.832 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: mon.smithi022@-1(???).mds e7 new map 2024-03-21T12:39:53.832 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: mon.smithi022@-1(???).mds e7 print_map 2024-03-21T12:39:53.833 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: e7 2024-03-21T12:39:53.833 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-03-21T12:39:53.833 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-03-21T12:39:53.833 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: legacy client fscid: 1 2024-03-21T12:39:53.833 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: 2024-03-21T12:39:53.833 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: Filesystem 'foofs' (1) 2024-03-21T12:39:53.833 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: fs_name foofs 2024-03-21T12:39:53.833 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: epoch 5 2024-03-21T12:39:53.833 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-03-21T12:39:53.834 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: created 2024-03-21T12:34:22.999525+0000 2024-03-21T12:39:53.834 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: modified 2024-03-21T12:34:29.019384+0000 2024-03-21T12:39:53.834 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: tableserver 0 2024-03-21T12:39:53.834 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: root 0 2024-03-21T12:39:53.834 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: session_timeout 60 2024-03-21T12:39:53.834 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: session_autoclose 300 2024-03-21T12:39:53.834 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: max_file_size 1099511627776 2024-03-21T12:39:53.834 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: max_xattr_size 65536 2024-03-21T12:39:53.834 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: required_client_features {} 2024-03-21T12:39:53.834 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: last_failure 0 2024-03-21T12:39:53.834 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: last_failure_osd_epoch 0 2024-03-21T12:39:53.834 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-03-21T12:39:53.835 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: max_mds 1 2024-03-21T12:39:53.835 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: in 0 2024-03-21T12:39:53.835 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: up {0=14488} 2024-03-21T12:39:53.835 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: failed 2024-03-21T12:39:53.835 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: damaged 2024-03-21T12:39:53.835 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: stopped 2024-03-21T12:39:53.835 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: data_pools [3] 2024-03-21T12:39:53.835 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: metadata_pool 2 2024-03-21T12:39:53.835 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: inline_data disabled 2024-03-21T12:39:53.835 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: balancer 2024-03-21T12:39:53.835 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: bal_rank_mask -1 2024-03-21T12:39:53.835 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: standby_count_wanted 1 2024-03-21T12:39:53.835 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: qdb_cluster leader: 0 members: 2024-03-21T12:39:53.835 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: [mds.foofs.smithi022.bcnxho{0:14488} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.22:6834/2456616276,v1:172.21.15.22:6835/2456616276] compat {c=[1],r=[1],i=[7ff]}] 2024-03-21T12:39:53.836 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: 2024-03-21T12:39:53.836 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: 2024-03-21T12:39:53.836 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: Standby daemons: 2024-03-21T12:39:53.836 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: 2024-03-21T12:39:53.836 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout: [mds.foofs.smithi189.lluvko{-1:24289} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.189:6832/2684579462,v1:172.21.15.189:6833/2684579462] compat {c=[1],r=[1],i=[7ff]}] 2024-03-21T12:39:53.836 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: mon.smithi022@-1(???).osd e72 crush map has features 3314933000854323200, adjusting msgr requires 2024-03-21T12:39:53.836 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: mon.smithi022@-1(???).osd e72 crush map has features 432629239337189376, adjusting msgr requires 2024-03-21T12:39:53.836 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: mon.smithi022@-1(???).osd e72 crush map has features 432629239337189376, adjusting msgr requires 2024-03-21T12:39:53.836 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: mon.smithi022@-1(???).osd e72 crush map has features 432629239337189376, adjusting msgr requires 2024-03-21T12:39:53.836 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 ceph-mon[67047]: mon.smithi022@-1(???).paxosservice(auth 1..29) refresh upgraded, format 0 -> 3 2024-03-21T12:39:53.856 INFO:teuthology.orchestra.run.smithi022.stdout: 5 151498 12.15 MB/sec execute 173 sec latency 62.911 ms 2024-03-21T12:39:54.051 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 bash[67028]: c620c9071b2c5e4c45ef86ebd2c1e7f031673bb2ec8844d2cdb9d9b86d552e78 2024-03-21T12:39:54.051 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:53 smithi022 systemd[1]: Started Ceph mon.smithi022 for 3fc7f0f2-e77e-11ee-95cd-87774f69a715. 2024-03-21T12:39:54.856 INFO:teuthology.orchestra.run.smithi022.stdout: 5 151896 12.09 MB/sec execute 174 sec latency 62.420 ms 2024-03-21T12:39:55.342 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:55 smithi022 ceph-mon[67047]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mon metadata", "id": "smithi022"}]: dispatch 2024-03-21T12:39:55.342 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:55 smithi022 ceph-mon[67047]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mon metadata", "id": "smithi189"}]: dispatch 2024-03-21T12:39:55.342 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:55 smithi022 ceph-mon[67047]: pgmap v33: 97 pgs: 97 active+clean; 341 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.4 MiB/s wr, 527 op/s 2024-03-21T12:39:55.342 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:55 smithi022 ceph-mon[67047]: pgmap v34: 97 pgs: 97 active+clean; 341 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.4 MiB/s wr, 527 op/s 2024-03-21T12:39:55.342 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:55 smithi022 ceph-mon[67047]: mon.smithi022 calling monitor election 2024-03-21T12:39:55.342 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:55 smithi022 ceph-mon[67047]: mon.smithi022 is new leader, mons smithi022,smithi189 in quorum (ranks 0,1) 2024-03-21T12:39:55.342 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:55 smithi022 ceph-mon[67047]: monmap e2: 2 mons at {smithi022=[v2:172.21.15.22:3300/0,v1:172.21.15.22:6789/0],smithi189=[v2:172.21.15.189:3300/0,v1:172.21.15.189:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-03-21T12:39:55.343 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:55 smithi022 ceph-mon[67047]: fsmap foofs:1 {0=foofs.smithi022.bcnxho=up:active} 1 up:standby 2024-03-21T12:39:55.343 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:55 smithi022 ceph-mon[67047]: osdmap e72: 8 total, 8 up, 8 in 2024-03-21T12:39:55.343 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:55 smithi022 ceph-mon[67047]: mgrmap e34: smithi022.fektzs(active, since 60s), standbys: smithi189.cnxzvu 2024-03-21T12:39:55.343 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:55 smithi022 ceph-mon[67047]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-03-21T12:39:55.343 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:55 smithi022 ceph-mon[67047]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-21T12:39:55.343 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:55 smithi022 ceph-mon[67047]: daemon grafana.smithi022 on smithi022 is in unknown state 2024-03-21T12:39:55.343 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:55 smithi022 ceph-mon[67047]: from='mgr.14768 ' entity='' 2024-03-21T12:39:55.343 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:55 smithi022 ceph-mon[67047]: mgrmap e35: smithi022.fektzs(active, since 60s), standbys: smithi189.cnxzvu 2024-03-21T12:39:55.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:55 smithi189 ceph-mon[27080]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mon metadata", "id": "smithi022"}]: dispatch 2024-03-21T12:39:55.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:55 smithi189 ceph-mon[27080]: from='mgr.14768 172.21.15.22:0/2106066878' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mon metadata", "id": "smithi189"}]: dispatch 2024-03-21T12:39:55.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:55 smithi189 ceph-mon[27080]: pgmap v33: 97 pgs: 97 active+clean; 341 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.4 MiB/s wr, 527 op/s 2024-03-21T12:39:55.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:55 smithi189 ceph-mon[27080]: pgmap v34: 97 pgs: 97 active+clean; 341 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.4 MiB/s wr, 527 op/s 2024-03-21T12:39:55.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:55 smithi189 ceph-mon[27080]: mon.smithi022 calling monitor election 2024-03-21T12:39:55.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:55 smithi189 ceph-mon[27080]: mon.smithi022 is new leader, mons smithi022,smithi189 in quorum (ranks 0,1) 2024-03-21T12:39:55.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:55 smithi189 ceph-mon[27080]: monmap e2: 2 mons at {smithi022=[v2:172.21.15.22:3300/0,v1:172.21.15.22:6789/0],smithi189=[v2:172.21.15.189:3300/0,v1:172.21.15.189:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-03-21T12:39:55.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:55 smithi189 ceph-mon[27080]: fsmap foofs:1 {0=foofs.smithi022.bcnxho=up:active} 1 up:standby 2024-03-21T12:39:55.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:55 smithi189 ceph-mon[27080]: osdmap e72: 8 total, 8 up, 8 in 2024-03-21T12:39:55.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:55 smithi189 ceph-mon[27080]: mgrmap e34: smithi022.fektzs(active, since 60s), standbys: smithi189.cnxzvu 2024-03-21T12:39:55.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:55 smithi189 ceph-mon[27080]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-03-21T12:39:55.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:55 smithi189 ceph-mon[27080]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-21T12:39:55.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:55 smithi189 ceph-mon[27080]: daemon grafana.smithi022 on smithi022 is in unknown state 2024-03-21T12:39:55.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:55 smithi189 ceph-mon[27080]: from='mgr.14768 ' entity='' 2024-03-21T12:39:55.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:55 smithi189 ceph-mon[27080]: mgrmap e35: smithi022.fektzs(active, since 60s), standbys: smithi189.cnxzvu 2024-03-21T12:39:55.856 INFO:teuthology.orchestra.run.smithi022.stdout: 5 152252 12.06 MB/sec execute 175 sec latency 38.050 ms 2024-03-21T12:39:56.859 INFO:teuthology.orchestra.run.smithi022.stdout: 5 152675 12.09 MB/sec execute 176 sec latency 45.041 ms 2024-03-21T12:39:57.379 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:39:57 smithi022 ceph-mon[67047]: mgrmap e36: smithi022.fektzs(active, since 61s), standbys: smithi189.cnxzvu 2024-03-21T12:39:57.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:39:57 smithi189 ceph-mon[27080]: mgrmap e36: smithi022.fektzs(active, since 61s), standbys: smithi189.cnxzvu 2024-03-21T12:39:57.856 INFO:teuthology.orchestra.run.smithi022.stdout: 5 153237 12.13 MB/sec execute 177 sec latency 34.342 ms 2024-03-21T12:39:58.856 INFO:teuthology.orchestra.run.smithi022.stdout: 5 153671 12.12 MB/sec execute 178 sec latency 25.563 ms 2024-03-21T12:39:59.856 INFO:teuthology.orchestra.run.smithi022.stdout: 5 154062 12.11 MB/sec execute 179 sec latency 24.913 ms 2024-03-21T12:40:00.126 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:00 smithi189 ceph-mon[27080]: overall HEALTH_WARN 1 failed cephadm daemon(s) 2024-03-21T12:40:00.146 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:00 smithi022 ceph-mon[67047]: overall HEALTH_WARN 1 failed cephadm daemon(s) 2024-03-21T12:40:00.856 INFO:teuthology.orchestra.run.smithi022.stdout: 5 154411 12.09 MB/sec execute 180 sec latency 41.604 ms 2024-03-21T12:40:01.856 INFO:teuthology.orchestra.run.smithi022.stdout: 5 154734 12.08 MB/sec execute 181 sec latency 58.398 ms 2024-03-21T12:40:02.856 INFO:teuthology.orchestra.run.smithi022.stdout: 5 155045 12.06 MB/sec execute 182 sec latency 73.206 ms 2024-03-21T12:40:03.478 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:03 smithi022 ceph-mon[67047]: Standby manager daemon smithi189.cnxzvu restarted 2024-03-21T12:40:03.479 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:03 smithi022 ceph-mon[67047]: Standby manager daemon smithi189.cnxzvu started 2024-03-21T12:40:03.479 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:03 smithi022 ceph-mon[67047]: from='mgr.? 172.21.15.189:0/510766100' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi189.cnxzvu/crt"}]: dispatch 2024-03-21T12:40:03.479 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:03 smithi022 ceph-mon[67047]: from='mgr.? 172.21.15.189:0/510766100' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-21T12:40:03.479 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:03 smithi022 ceph-mon[67047]: from='mgr.? 172.21.15.189:0/510766100' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi189.cnxzvu/key"}]: dispatch 2024-03-21T12:40:03.479 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:03 smithi022 ceph-mon[67047]: from='mgr.? 172.21.15.189:0/510766100' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-21T12:40:03.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:03 smithi189 ceph-mon[27080]: Standby manager daemon smithi189.cnxzvu restarted 2024-03-21T12:40:03.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:03 smithi189 ceph-mon[27080]: Standby manager daemon smithi189.cnxzvu started 2024-03-21T12:40:03.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:03 smithi189 ceph-mon[27080]: from='mgr.? 172.21.15.189:0/510766100' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi189.cnxzvu/crt"}]: dispatch 2024-03-21T12:40:03.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:03 smithi189 ceph-mon[27080]: from='mgr.? 172.21.15.189:0/510766100' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-21T12:40:03.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:03 smithi189 ceph-mon[27080]: from='mgr.? 172.21.15.189:0/510766100' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi189.cnxzvu/key"}]: dispatch 2024-03-21T12:40:03.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:03 smithi189 ceph-mon[27080]: from='mgr.? 172.21.15.189:0/510766100' entity='mgr.smithi189.cnxzvu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-21T12:40:03.856 INFO:teuthology.orchestra.run.smithi022.stdout: 5 155598 12.04 MB/sec execute 183 sec latency 34.761 ms 2024-03-21T12:40:04.486 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:04 smithi022 ceph-mon[67047]: mgrmap e37: smithi022.fektzs(active, since 68s), standbys: smithi189.cnxzvu 2024-03-21T12:40:04.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:04 smithi189 ceph-mon[27080]: mgrmap e37: smithi022.fektzs(active, since 68s), standbys: smithi189.cnxzvu 2024-03-21T12:40:04.856 INFO:teuthology.orchestra.run.smithi022.stdout: 5 156027 12.05 MB/sec execute 184 sec latency 27.700 ms 2024-03-21T12:40:05.749 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:05 smithi022 ceph-mon[67047]: Active manager daemon smithi022.fektzs restarted 2024-03-21T12:40:05.749 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:05 smithi022 ceph-mon[67047]: Activating manager daemon smithi022.fektzs 2024-03-21T12:40:05.749 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:05 smithi022 ceph-mon[67047]: osdmap e73: 8 total, 8 up, 8 in 2024-03-21T12:40:05.749 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:05 smithi022 ceph-mon[67047]: mgrmap e38: smithi022.fektzs(active, starting, since 0.0278087s), standbys: smithi189.cnxzvu 2024-03-21T12:40:05.749 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:05 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mon metadata", "id": "smithi022"}]: dispatch 2024-03-21T12:40:05.749 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:05 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mon metadata", "id": "smithi189"}]: dispatch 2024-03-21T12:40:05.749 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:05 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi022.bcnxho"}]: dispatch 2024-03-21T12:40:05.749 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:05 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi189.lluvko"}]: dispatch 2024-03-21T12:40:05.750 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:05 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mgr metadata", "who": "smithi022.fektzs", "id": "smithi022.fektzs"}]: dispatch 2024-03-21T12:40:05.750 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:05 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mgr metadata", "who": "smithi189.cnxzvu", "id": "smithi189.cnxzvu"}]: dispatch 2024-03-21T12:40:05.750 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:05 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-21T12:40:05.750 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:05 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-21T12:40:05.750 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:05 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-21T12:40:05.750 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:05 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-21T12:40:05.750 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:05 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-21T12:40:05.750 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:05 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-21T12:40:05.750 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:05 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-21T12:40:05.750 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:05 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-21T12:40:05.750 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:05 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-21T12:40:05.750 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:05 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-21T12:40:05.750 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:05 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-21T12:40:05.856 INFO:teuthology.orchestra.run.smithi022.stdout: 5 156484 12.07 MB/sec execute 185 sec latency 52.995 ms 2024-03-21T12:40:05.920 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:05 smithi189 ceph-mon[27080]: Active manager daemon smithi022.fektzs restarted 2024-03-21T12:40:05.920 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:05 smithi189 ceph-mon[27080]: Activating manager daemon smithi022.fektzs 2024-03-21T12:40:05.920 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:05 smithi189 ceph-mon[27080]: osdmap e73: 8 total, 8 up, 8 in 2024-03-21T12:40:05.920 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:05 smithi189 ceph-mon[27080]: mgrmap e38: smithi022.fektzs(active, starting, since 0.0278087s), standbys: smithi189.cnxzvu 2024-03-21T12:40:05.920 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:05 smithi189 ceph-mon[27080]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mon metadata", "id": "smithi022"}]: dispatch 2024-03-21T12:40:05.920 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:05 smithi189 ceph-mon[27080]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mon metadata", "id": "smithi189"}]: dispatch 2024-03-21T12:40:05.920 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:05 smithi189 ceph-mon[27080]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi022.bcnxho"}]: dispatch 2024-03-21T12:40:05.920 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:05 smithi189 ceph-mon[27080]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi189.lluvko"}]: dispatch 2024-03-21T12:40:05.920 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:05 smithi189 ceph-mon[27080]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mgr metadata", "who": "smithi022.fektzs", "id": "smithi022.fektzs"}]: dispatch 2024-03-21T12:40:05.920 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:05 smithi189 ceph-mon[27080]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mgr metadata", "who": "smithi189.cnxzvu", "id": "smithi189.cnxzvu"}]: dispatch 2024-03-21T12:40:05.921 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:05 smithi189 ceph-mon[27080]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-21T12:40:05.921 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:05 smithi189 ceph-mon[27080]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-21T12:40:05.921 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:05 smithi189 ceph-mon[27080]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-21T12:40:05.921 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:05 smithi189 ceph-mon[27080]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-21T12:40:05.921 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:05 smithi189 ceph-mon[27080]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-21T12:40:05.921 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:05 smithi189 ceph-mon[27080]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-21T12:40:05.921 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:05 smithi189 ceph-mon[27080]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-21T12:40:05.921 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:05 smithi189 ceph-mon[27080]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-21T12:40:05.921 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:05 smithi189 ceph-mon[27080]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-21T12:40:05.921 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:05 smithi189 ceph-mon[27080]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-21T12:40:05.921 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:05 smithi189 ceph-mon[27080]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-21T12:40:06.654 INFO:teuthology.orchestra.run.smithi022.stdout:true 2024-03-21T12:40:06.738 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:06 smithi022 ceph-mon[67047]: Manager daemon smithi022.fektzs is now available 2024-03-21T12:40:06.738 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:06 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:40:06.738 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:06 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:40:06.739 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:06 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi022.fektzs/mirror_snapshot_schedule"}]: dispatch 2024-03-21T12:40:06.739 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:06 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi022.fektzs/trash_purge_schedule"}]: dispatch 2024-03-21T12:40:06.739 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:06 smithi022 ceph-mon[67047]: [21/Mar/2024:12:40:06] ENGINE Bus STARTING 2024-03-21T12:40:06.859 INFO:teuthology.orchestra.run.smithi022.stdout: 5 156842 12.06 MB/sec execute 186 sec latency 61.591 ms 2024-03-21T12:40:07.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:06 smithi189 ceph-mon[27080]: Manager daemon smithi022.fektzs is now available 2024-03-21T12:40:07.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:06 smithi189 ceph-mon[27080]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:40:07.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:06 smithi189 ceph-mon[27080]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:40:07.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:06 smithi189 ceph-mon[27080]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi022.fektzs/mirror_snapshot_schedule"}]: dispatch 2024-03-21T12:40:07.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:06 smithi189 ceph-mon[27080]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi022.fektzs/trash_purge_schedule"}]: dispatch 2024-03-21T12:40:07.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:06 smithi189 ceph-mon[27080]: [21/Mar/2024:12:40:06] ENGINE Bus STARTING 2024-03-21T12:40:07.724 INFO:teuthology.orchestra.run.smithi022.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-21T12:40:07.724 INFO:teuthology.orchestra.run.smithi022.stdout:alertmanager.smithi022 smithi022 *:9093,9094 running (2m) 23s ago 11m 25.7M - 0.23.0 ba2b418f427c 14eb2a3ca318 2024-03-21T12:40:07.724 INFO:teuthology.orchestra.run.smithi022.stdout:crash.smithi022 smithi022 running (11m) 23s ago 11m 6405k - 17.2.0 e1d6a67b021e 47deb70fd250 2024-03-21T12:40:07.725 INFO:teuthology.orchestra.run.smithi022.stdout:crash.smithi189 smithi189 running (10m) 29s ago 10m 6405k - 17.2.0 e1d6a67b021e 9a2e0e9b1809 2024-03-21T12:40:07.725 INFO:teuthology.orchestra.run.smithi022.stdout:grafana.smithi022 smithi022 *:3000 unknown 23s ago 11m - - dad864ee21e9 194c0f64bae3 2024-03-21T12:40:07.725 INFO:teuthology.orchestra.run.smithi022.stdout:mds.foofs.smithi022.bcnxho smithi022 running (5m) 23s ago 5m 37.3M - 17.2.0 e1d6a67b021e db26b67b8ee7 2024-03-21T12:40:07.725 INFO:teuthology.orchestra.run.smithi022.stdout:mds.foofs.smithi189.lluvko smithi189 running (5m) 29s ago 5m 16.4M - 17.2.0 e1d6a67b021e cf153aff1387 2024-03-21T12:40:07.725 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.smithi022.fektzs smithi022 *:8443,9283,8765 running (94s) 23s ago 12m 494M - 19.0.0-2267-g960fbbe4 19e12293512f d57cebd306b3 2024-03-21T12:40:07.725 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.smithi189.cnxzvu smithi189 *:8443,9283,8765 running (35s) 29s ago 10m 93.1M - 19.0.0-2267-g960fbbe4 19e12293512f 41663955c56f 2024-03-21T12:40:07.725 INFO:teuthology.orchestra.run.smithi022.stdout:mon.smithi022 smithi022 running (12m) 23s ago 12m 55.4M 2048M 17.2.0 e1d6a67b021e 0d248775e0e5 2024-03-21T12:40:07.725 INFO:teuthology.orchestra.run.smithi022.stdout:mon.smithi189 smithi189 running (10m) 29s ago 10m 44.9M 2048M 17.2.0 e1d6a67b021e 82da5f3cd49a 2024-03-21T12:40:07.725 INFO:teuthology.orchestra.run.smithi022.stdout:nfs.foo.0.0.smithi022.ynhbbq smithi022 *:2049 running (5m) 23s ago 5m 256M - 4.0 e1d6a67b021e 33e48e64c60b 2024-03-21T12:40:07.725 INFO:teuthology.orchestra.run.smithi022.stdout:nfs.foo.1.0.smithi189.yzsqml smithi189 *:2049 running (5m) 29s ago 5m 83.1M - 4.0 e1d6a67b021e 48fab2cbdde2 2024-03-21T12:40:07.725 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.smithi022 smithi022 *:9100 running (2m) 23s ago 11m 17.4M - 1.3.1 1dbe0e931976 33287f803f4f 2024-03-21T12:40:07.725 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.smithi189 smithi189 *:9100 running (106s) 29s ago 10m 10.1M - 1.3.1 1dbe0e931976 b51d872e438a 2024-03-21T12:40:07.726 INFO:teuthology.orchestra.run.smithi022.stdout:osd.0 smithi022 running (10m) 23s ago 10m 502M 1937M 17.2.0 e1d6a67b021e 54e6631bd10f 2024-03-21T12:40:07.726 INFO:teuthology.orchestra.run.smithi022.stdout:osd.1 smithi022 running (9m) 23s ago 9m 427M 1937M 17.2.0 e1d6a67b021e 68165c5d09cb 2024-03-21T12:40:07.726 INFO:teuthology.orchestra.run.smithi022.stdout:osd.2 smithi022 running (9m) 23s ago 9m 316M 1937M 17.2.0 e1d6a67b021e fbec9bef8433 2024-03-21T12:40:07.726 INFO:teuthology.orchestra.run.smithi022.stdout:osd.3 smithi022 running (8m) 23s ago 8m 464M 1937M 17.2.0 e1d6a67b021e b1462ffc1b9e 2024-03-21T12:40:07.726 INFO:teuthology.orchestra.run.smithi022.stdout:osd.4 smithi189 running (8m) 29s ago 8m 482M 2704M 17.2.0 e1d6a67b021e 1bedc4eb02ed 2024-03-21T12:40:07.726 INFO:teuthology.orchestra.run.smithi022.stdout:osd.5 smithi189 running (7m) 29s ago 7m 551M 2704M 17.2.0 e1d6a67b021e 25fbab9786a5 2024-03-21T12:40:07.726 INFO:teuthology.orchestra.run.smithi022.stdout:osd.6 smithi189 running (7m) 29s ago 7m 455M 2704M 17.2.0 e1d6a67b021e 0c192fcfd65b 2024-03-21T12:40:07.726 INFO:teuthology.orchestra.run.smithi022.stdout:osd.7 smithi189 running (7m) 29s ago 7m 365M 2704M 17.2.0 e1d6a67b021e 3d0723e1f863 2024-03-21T12:40:07.726 INFO:teuthology.orchestra.run.smithi022.stdout:prometheus.smithi022 smithi022 *:9095 running (41s) 23s ago 10m 50.4M - 2.33.4 514e6a882f6e 8171bbe41d2c 2024-03-21T12:40:07.762 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:07 smithi022 ceph-mon[67047]: [21/Mar/2024:12:40:06] ENGINE Serving on https://172.21.15.22:7150 2024-03-21T12:40:07.763 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:07 smithi022 ceph-mon[67047]: [21/Mar/2024:12:40:06] ENGINE Serving on http://172.21.15.22:8765 2024-03-21T12:40:07.763 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:07 smithi022 ceph-mon[67047]: [21/Mar/2024:12:40:06] ENGINE Bus STARTED 2024-03-21T12:40:07.763 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:07 smithi022 ceph-mon[67047]: mgrmap e39: smithi022.fektzs(active, since 1.06922s), standbys: smithi189.cnxzvu 2024-03-21T12:40:07.763 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:07 smithi022 ceph-mon[67047]: from='client.34106 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:40:07.763 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:07 smithi022 ceph-mon[67047]: pgmap v3: 97 pgs: 97 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-03-21T12:40:07.763 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:07 smithi022 ceph-mon[67047]: from='client.34130 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:40:07.856 INFO:teuthology.orchestra.run.smithi022.stdout: 5 157141 12.04 MB/sec execute 187 sec latency 49.699 ms 2024-03-21T12:40:07.896 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:07 smithi189 ceph-mon[27080]: [21/Mar/2024:12:40:06] ENGINE Serving on https://172.21.15.22:7150 2024-03-21T12:40:07.896 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:07 smithi189 ceph-mon[27080]: [21/Mar/2024:12:40:06] ENGINE Serving on http://172.21.15.22:8765 2024-03-21T12:40:07.896 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:07 smithi189 ceph-mon[27080]: [21/Mar/2024:12:40:06] ENGINE Bus STARTED 2024-03-21T12:40:07.896 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:07 smithi189 ceph-mon[27080]: mgrmap e39: smithi022.fektzs(active, since 1.06922s), standbys: smithi189.cnxzvu 2024-03-21T12:40:07.896 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:07 smithi189 ceph-mon[27080]: from='client.34106 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:40:07.896 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:07 smithi189 ceph-mon[27080]: pgmap v3: 97 pgs: 97 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-03-21T12:40:07.896 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:07 smithi189 ceph-mon[27080]: from='client.34130 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:40:08.332 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-03-21T12:40:08.332 INFO:teuthology.orchestra.run.smithi022.stdout: "mon": { 2024-03-21T12:40:08.332 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-03-21T12:40:08.332 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 1 2024-03-21T12:40:08.332 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:40:08.332 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr": { 2024-03-21T12:40:08.332 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 2 2024-03-21T12:40:08.332 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:40:08.332 INFO:teuthology.orchestra.run.smithi022.stdout: "osd": { 2024-03-21T12:40:08.332 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-03-21T12:40:08.332 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:40:08.332 INFO:teuthology.orchestra.run.smithi022.stdout: "mds": { 2024-03-21T12:40:08.332 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-21T12:40:08.333 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:40:08.333 INFO:teuthology.orchestra.run.smithi022.stdout: "overall": { 2024-03-21T12:40:08.333 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 11, 2024-03-21T12:40:08.333 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 3 2024-03-21T12:40:08.333 INFO:teuthology.orchestra.run.smithi022.stdout: } 2024-03-21T12:40:08.333 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-03-21T12:40:08.794 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:08 smithi022 ceph-mon[67047]: pgmap v4: 97 pgs: 97 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-03-21T12:40:08.795 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:08 smithi022 ceph-mon[67047]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-03-21T12:40:08.795 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:08 smithi022 ceph-mon[67047]: Cluster is now healthy 2024-03-21T12:40:08.795 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:08 smithi022 ceph-mon[67047]: mgrmap e40: smithi022.fektzs(active, since 2s), standbys: smithi189.cnxzvu 2024-03-21T12:40:08.795 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:08 smithi022 ceph-mon[67047]: from='client.24633 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:40:08.795 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:08 smithi022 ceph-mon[67047]: from='client.? 172.21.15.22:0/3686363642' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:40:08.818 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-03-21T12:40:08.819 INFO:teuthology.orchestra.run.smithi022.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93", 2024-03-21T12:40:08.819 INFO:teuthology.orchestra.run.smithi022.stdout: "in_progress": true, 2024-03-21T12:40:08.819 INFO:teuthology.orchestra.run.smithi022.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-21T12:40:08.819 INFO:teuthology.orchestra.run.smithi022.stdout: "services_complete": [ 2024-03-21T12:40:08.819 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr" 2024-03-21T12:40:08.819 INFO:teuthology.orchestra.run.smithi022.stdout: ], 2024-03-21T12:40:08.819 INFO:teuthology.orchestra.run.smithi022.stdout: "progress": "2/23 daemons upgraded", 2024-03-21T12:40:08.819 INFO:teuthology.orchestra.run.smithi022.stdout: "message": "", 2024-03-21T12:40:08.819 INFO:teuthology.orchestra.run.smithi022.stdout: "is_paused": false 2024-03-21T12:40:08.819 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-03-21T12:40:08.859 INFO:teuthology.orchestra.run.smithi022.stdout: 5 157318 12.00 MB/sec execute 188 sec latency 37.230 ms 2024-03-21T12:40:08.931 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:08 smithi189 ceph-mon[27080]: pgmap v4: 97 pgs: 97 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-03-21T12:40:08.931 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:08 smithi189 ceph-mon[27080]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-03-21T12:40:08.931 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:08 smithi189 ceph-mon[27080]: Cluster is now healthy 2024-03-21T12:40:08.931 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:08 smithi189 ceph-mon[27080]: mgrmap e40: smithi022.fektzs(active, since 2s), standbys: smithi189.cnxzvu 2024-03-21T12:40:08.931 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:08 smithi189 ceph-mon[27080]: from='client.24633 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:40:08.931 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:08 smithi189 ceph-mon[27080]: from='client.? 172.21.15.22:0/3686363642' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:40:09.429 INFO:teuthology.orchestra.run.smithi022.stdout:HEALTH_OK 2024-03-21T12:40:09.801 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:09 smithi022 ceph-mon[67047]: from='client.34144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:40:09.801 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:09 smithi022 ceph-mon[67047]: from='client.? 172.21.15.22:0/2637546589' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-21T12:40:09.856 INFO:teuthology.orchestra.run.smithi022.stdout: 5 157741 11.99 MB/sec execute 189 sec latency 38.038 ms 2024-03-21T12:40:09.937 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:09 smithi189 ceph-mon[27080]: from='client.34144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:40:09.937 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:09 smithi189 ceph-mon[27080]: from='client.? 172.21.15.22:0/2637546589' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-21T12:40:10.823 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:10 smithi022 ceph-mon[67047]: pgmap v5: 97 pgs: 97 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-03-21T12:40:10.824 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:10 smithi022 ceph-mon[67047]: mgrmap e41: smithi022.fektzs(active, since 4s), standbys: smithi189.cnxzvu 2024-03-21T12:40:10.824 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:10 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:10.824 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:10 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:10.856 INFO:teuthology.orchestra.run.smithi022.stdout: 5 158186 12.00 MB/sec execute 190 sec latency 61.594 ms 2024-03-21T12:40:11.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:10 smithi189 ceph-mon[27080]: pgmap v5: 97 pgs: 97 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-03-21T12:40:11.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:10 smithi189 ceph-mon[27080]: mgrmap e41: smithi022.fektzs(active, since 4s), standbys: smithi189.cnxzvu 2024-03-21T12:40:11.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:10 smithi189 ceph-mon[27080]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:11.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:10 smithi189 ceph-mon[27080]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:11.856 INFO:teuthology.orchestra.run.smithi022.stdout: 5 158606 11.99 MB/sec execute 191 sec latency 38.384 ms 2024-03-21T12:40:12.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:12 smithi022 ceph-mon[67047]: pgmap v6: 97 pgs: 97 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-03-21T12:40:12.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:12 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:12.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:12 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:12.856 INFO:teuthology.orchestra.run.smithi022.stdout: 5 159120 11.97 MB/sec execute 192 sec latency 53.915 ms 2024-03-21T12:40:13.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:12 smithi189 ceph-mon[27080]: pgmap v6: 97 pgs: 97 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-03-21T12:40:13.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:12 smithi189 ceph-mon[27080]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:13.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:12 smithi189 ceph-mon[27080]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:13.856 INFO:teuthology.orchestra.run.smithi022.stdout: 5 159444 11.95 MB/sec execute 193 sec latency 78.177 ms 2024-03-21T12:40:14.856 INFO:teuthology.orchestra.run.smithi022.stdout: 5 159926 11.99 MB/sec execute 194 sec latency 43.873 ms 2024-03-21T12:40:14.864 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:14 smithi022 ceph-mon[67047]: pgmap v7: 97 pgs: 97 active+clean; 337 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 35 KiB/s rd, 4.2 MiB/s wr, 404 op/s 2024-03-21T12:40:15.101 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:14 smithi189 ceph-mon[27080]: pgmap v7: 97 pgs: 97 active+clean; 337 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 35 KiB/s rd, 4.2 MiB/s wr, 404 op/s 2024-03-21T12:40:15.856 INFO:teuthology.orchestra.run.smithi022.stdout: 5 160450 12.00 MB/sec execute 195 sec latency 43.159 ms 2024-03-21T12:40:16.209 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:16 smithi189 ceph-mon[27080]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:16.209 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:16 smithi189 ceph-mon[27080]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:16.393 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:16 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:16.394 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:16 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:16.857 INFO:teuthology.orchestra.run.smithi022.stdout: 5 160767 11.98 MB/sec execute 196 sec latency 33.467 ms 2024-03-21T12:40:17.409 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:17 smithi022 ceph-mon[67047]: pgmap v8: 97 pgs: 97 active+clean; 337 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 27 KiB/s rd, 3.3 MiB/s wr, 313 op/s 2024-03-21T12:40:17.409 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:17 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:17.409 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:17 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:17.409 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:17 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:17.409 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:17 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:17.409 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:17 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:40:17.409 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:17 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:40:17.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:17 smithi189 ceph-mon[27080]: pgmap v8: 97 pgs: 97 active+clean; 337 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 27 KiB/s rd, 3.3 MiB/s wr, 313 op/s 2024-03-21T12:40:17.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:17 smithi189 ceph-mon[27080]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:17.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:17 smithi189 ceph-mon[27080]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:17.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:17 smithi189 ceph-mon[27080]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:17.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:17 smithi189 ceph-mon[27080]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:17.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:17 smithi189 ceph-mon[27080]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:40:17.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:17 smithi189 ceph-mon[27080]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:40:17.859 INFO:teuthology.orchestra.run.smithi022.stdout: 5 161059 11.97 MB/sec execute 197 sec latency 40.159 ms 2024-03-21T12:40:18.169 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:18 smithi022 ceph-mon[67047]: Updating smithi022:/etc/ceph/ceph.conf 2024-03-21T12:40:18.169 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:18 smithi022 ceph-mon[67047]: Updating smithi189:/etc/ceph/ceph.conf 2024-03-21T12:40:18.169 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:18 smithi022 ceph-mon[67047]: Updating smithi022:/var/lib/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/config/ceph.conf 2024-03-21T12:40:18.169 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:18 smithi022 ceph-mon[67047]: Updating smithi189:/var/lib/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/config/ceph.conf 2024-03-21T12:40:18.169 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:18 smithi022 ceph-mon[67047]: Updating smithi022:/etc/ceph/ceph.client.admin.keyring 2024-03-21T12:40:18.169 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:18 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:18.169 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:18 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:18.169 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:18 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:18.170 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:18 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:18.170 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:18 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:18.170 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:18 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:18.170 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:18 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:40:18.170 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:18 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:40:18.170 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:18 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-03-21T12:40:18.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:18 smithi189 ceph-mon[27080]: Updating smithi022:/etc/ceph/ceph.conf 2024-03-21T12:40:18.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:18 smithi189 ceph-mon[27080]: Updating smithi189:/etc/ceph/ceph.conf 2024-03-21T12:40:18.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:18 smithi189 ceph-mon[27080]: Updating smithi022:/var/lib/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/config/ceph.conf 2024-03-21T12:40:18.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:18 smithi189 ceph-mon[27080]: Updating smithi189:/var/lib/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/config/ceph.conf 2024-03-21T12:40:18.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:18 smithi189 ceph-mon[27080]: Updating smithi022:/etc/ceph/ceph.client.admin.keyring 2024-03-21T12:40:18.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:18 smithi189 ceph-mon[27080]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:18.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:18 smithi189 ceph-mon[27080]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:18.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:18 smithi189 ceph-mon[27080]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:18.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:18 smithi189 ceph-mon[27080]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:18.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:18 smithi189 ceph-mon[27080]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:18.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:18 smithi189 ceph-mon[27080]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:18.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:18 smithi189 ceph-mon[27080]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:40:18.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:18 smithi189 ceph-mon[27080]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:40:18.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:18 smithi189 ceph-mon[27080]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-03-21T12:40:18.856 INFO:teuthology.orchestra.run.smithi022.stdout: 5 161453 11.96 MB/sec execute 198 sec latency 42.900 ms 2024-03-21T12:40:19.194 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:19 smithi022 ceph-mon[67047]: Updating smithi189:/etc/ceph/ceph.client.admin.keyring 2024-03-21T12:40:19.195 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:19 smithi022 ceph-mon[67047]: Updating smithi022:/var/lib/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/config/ceph.client.admin.keyring 2024-03-21T12:40:19.195 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:19 smithi022 ceph-mon[67047]: pgmap v9: 97 pgs: 97 active+clean; 335 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 26 KiB/s rd, 5.4 MiB/s wr, 535 op/s 2024-03-21T12:40:19.195 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:19 smithi022 ceph-mon[67047]: Updating smithi189:/var/lib/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/config/ceph.client.admin.keyring 2024-03-21T12:40:19.195 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:19 smithi022 ceph-mon[67047]: pgmap v10: 97 pgs: 97 active+clean; 335 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 27 KiB/s rd, 5.7 MiB/s wr, 564 op/s 2024-03-21T12:40:19.195 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:19 smithi022 ceph-mon[67047]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-03-21T12:40:19.430 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:19 smithi189 ceph-mon[27080]: Updating smithi189:/etc/ceph/ceph.client.admin.keyring 2024-03-21T12:40:19.430 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:19 smithi189 ceph-mon[27080]: Updating smithi022:/var/lib/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/config/ceph.client.admin.keyring 2024-03-21T12:40:19.431 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:19 smithi189 ceph-mon[27080]: pgmap v9: 97 pgs: 97 active+clean; 335 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 26 KiB/s rd, 5.4 MiB/s wr, 535 op/s 2024-03-21T12:40:19.431 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:19 smithi189 ceph-mon[27080]: Updating smithi189:/var/lib/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/config/ceph.client.admin.keyring 2024-03-21T12:40:19.431 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:19 smithi189 ceph-mon[27080]: pgmap v10: 97 pgs: 97 active+clean; 335 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 27 KiB/s rd, 5.7 MiB/s wr, 564 op/s 2024-03-21T12:40:19.431 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:19 smithi189 ceph-mon[27080]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-03-21T12:40:19.857 INFO:teuthology.orchestra.run.smithi022.stdout: 5 161729 11.94 MB/sec execute 199 sec latency 48.225 ms 2024-03-21T12:40:20.857 INFO:teuthology.orchestra.run.smithi022.stdout: 5 162016 11.93 MB/sec execute 200 sec latency 92.188 ms 2024-03-21T12:40:21.232 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:21 smithi022 ceph-mon[67047]: pgmap v11: 97 pgs: 97 active+clean; 335 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 27 KiB/s rd, 5.7 MiB/s wr, 564 op/s 2024-03-21T12:40:21.233 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:21 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:21.233 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:21 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-21T12:40:21.233 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:21 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-21T12:40:21.233 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:21 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:40:21.233 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:21 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:40:21.517 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:21 smithi189 ceph-mon[27080]: pgmap v11: 97 pgs: 97 active+clean; 335 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 27 KiB/s rd, 5.7 MiB/s wr, 564 op/s 2024-03-21T12:40:21.517 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:21 smithi189 ceph-mon[27080]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:21.517 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:21 smithi189 ceph-mon[27080]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-21T12:40:21.517 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:21 smithi189 ceph-mon[27080]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-21T12:40:21.517 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:21 smithi189 ceph-mon[27080]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:40:21.517 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:21 smithi189 ceph-mon[27080]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:40:21.857 INFO:teuthology.orchestra.run.smithi022.stdout: 5 162307 11.89 MB/sec execute 201 sec latency 95.571 ms 2024-03-21T12:40:22.252 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:22 smithi022 ceph-mon[67047]: Upgrade: Updating mon.smithi189 2024-03-21T12:40:22.252 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:22 smithi022 ceph-mon[67047]: Deploying daemon mon.smithi189 on smithi189 2024-03-21T12:40:22.494 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:22 smithi189 ceph-mon[27080]: Upgrade: Updating mon.smithi189 2024-03-21T12:40:22.494 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:22 smithi189 ceph-mon[27080]: Deploying daemon mon.smithi189 on smithi189 2024-03-21T12:40:22.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:22 smithi189 systemd[1]: Stopping Ceph mon.smithi189 for 3fc7f0f2-e77e-11ee-95cd-87774f69a715... 2024-03-21T12:40:22.857 INFO:teuthology.orchestra.run.smithi022.stdout: 5 162653 11.85 MB/sec execute 202 sec latency 60.628 ms 2024-03-21T12:40:23.408 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:23 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-mon-smithi189[27064]: 2024-03-21T12:40:23.041+0000 7f574f5af700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi189 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-03-21T12:40:23.408 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:23 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-mon-smithi189[27064]: 2024-03-21T12:40:23.041+0000 7f574f5af700 -1 mon.smithi189@1(peon) e2 *** Got Signal Terminated *** 2024-03-21T12:40:23.408 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:23 smithi189 podman[53005]: 2024-03-21 12:40:23.08336186 +0000 UTC m=+0.179804755 container died 82da5f3cd49af448c49eb31cce9db07d3ca98de43c44a0d84682d3c551b825a2 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-mon-smithi189, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, distribution-scope=public, io.openshift.expose-services=, name=centos-stream, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, io.buildah.version=1.19.8, vcs-type=git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD) 2024-03-21T12:40:23.744 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:23 smithi189 podman[53005]: 2024-03-21 12:40:23.483867272 +0000 UTC m=+0.580310162 container cleanup 82da5f3cd49af448c49eb31cce9db07d3ca98de43c44a0d84682d3c551b825a2 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-mon-smithi189, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., distribution-scope=public, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, vcs-type=git, name=centos-stream, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, io.openshift.expose-services=, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-03-21T12:40:23.744 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:23 smithi189 bash[53005]: ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-mon-smithi189 2024-03-21T12:40:23.857 INFO:teuthology.orchestra.run.smithi022.stdout: 5 162890 11.81 MB/sec execute 203 sec latency 38.240 ms 2024-03-21T12:40:24.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:23 smithi189 podman[53017]: 2024-03-21 12:40:23.744405108 +0000 UTC m=+0.659516656 container remove 82da5f3cd49af448c49eb31cce9db07d3ca98de43c44a0d84682d3c551b825a2 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-mon-smithi189, GIT_BRANCH=HEAD, GIT_CLEAN=True, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , RELEASE=HEAD, vcs-type=git, version=8, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc.) 2024-03-21T12:40:24.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:23 smithi189 systemd[1]: ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715@mon.smithi189.service: Deactivated successfully. 2024-03-21T12:40:24.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:23 smithi189 systemd[1]: Stopped Ceph mon.smithi189 for 3fc7f0f2-e77e-11ee-95cd-87774f69a715. 2024-03-21T12:40:24.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:23 smithi189 systemd[1]: ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715@mon.smithi189.service: Consumed 7.156s CPU time. 2024-03-21T12:40:24.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 systemd[1]: Starting Ceph mon.smithi189 for 3fc7f0f2-e77e-11ee-95cd-87774f69a715... 2024-03-21T12:40:24.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 podman[53141]: 2024-03-21 12:40:24.327450748 +0000 UTC m=+0.023686646 image pull 19e12293512ff7978c8adb5f72a829060dd47f07bfd6903b6b3178d2ebc29e4e quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93 2024-03-21T12:40:24.858 INFO:teuthology.orchestra.run.smithi022.stdout: 5 163213 11.82 MB/sec execute 204 sec latency 44.115 ms 2024-03-21T12:40:24.915 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 podman[53141]: 2024-03-21 12:40:24.663978792 +0000 UTC m=+0.360214674 container create b353af3d8f3b6abef504d9964905f2616d9f4e964d160c9c7924d0d1ff28e456 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-mon-smithi189, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240319, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, GIT_CLEAN=True, ceph=True) 2024-03-21T12:40:24.915 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 podman[53141]: 2024-03-21 12:40:24.884739948 +0000 UTC m=+0.580975831 container init b353af3d8f3b6abef504d9964905f2616d9f4e964d160c9c7924d0d1ff28e456 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-mon-smithi189, GIT_BRANCH=HEAD, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240319, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-03-21T12:40:24.915 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 podman[53141]: 2024-03-21 12:40:24.888973981 +0000 UTC m=+0.585209866 container start b353af3d8f3b6abef504d9964905f2616d9f4e964d160c9c7924d0d1ff28e456 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-mon-smithi189, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240319, io.buildah.version=1.33.5, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, ceph=True, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.license=GPLv2) 2024-03-21T12:40:25.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: set uid:gid to 167:167 (ceph:ceph) 2024-03-21T12:40:25.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev), process ceph-mon, pid 2 2024-03-21T12:40:25.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: pidfile_write: ignore empty --pid-file 2024-03-21T12:40:25.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: load: jerasure load: lrc 2024-03-21T12:40:25.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: RocksDB version: 7.9.2 2024-03-21T12:40:25.287 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Git sha 0 2024-03-21T12:40:25.287 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Compile date 2024-03-21 08:53:26 2024-03-21T12:40:25.287 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: DB SUMMARY 2024-03-21T12:40:25.287 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: DB Session ID: EIS9PBMYDIWINWR17BXW 2024-03-21T12:40:25.287 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: CURRENT file: CURRENT 2024-03-21T12:40:25.287 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: IDENTITY file: IDENTITY 2024-03-21T12:40:25.287 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: MANIFEST file: MANIFEST-000009 size: 903 Bytes 2024-03-21T12:40:25.287 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi189/store.db dir, Total Num: 1, files: 000024.sst 2024-03-21T12:40:25.287 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi189/store.db: 000022.log size: 7224193 ; 2024-03-21T12:40:25.287 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.error_if_exists: 0 2024-03-21T12:40:25.287 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.create_if_missing: 0 2024-03-21T12:40:25.287 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.paranoid_checks: 1 2024-03-21T12:40:25.287 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.flush_verify_memtable_count: 1 2024-03-21T12:40:25.287 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-03-21T12:40:25.287 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-03-21T12:40:25.288 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.env: 0x55f02f3fdb20 2024-03-21T12:40:25.288 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.fs: PosixFileSystem 2024-03-21T12:40:25.288 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.info_log: 0x55f0311b97c0 2024-03-21T12:40:25.288 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.max_file_opening_threads: 16 2024-03-21T12:40:25.288 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.statistics: (nil) 2024-03-21T12:40:25.288 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.use_fsync: 0 2024-03-21T12:40:25.288 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.max_log_file_size: 0 2024-03-21T12:40:25.288 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-03-21T12:40:25.288 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.log_file_time_to_roll: 0 2024-03-21T12:40:25.288 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.keep_log_file_num: 1000 2024-03-21T12:40:25.288 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.recycle_log_file_num: 0 2024-03-21T12:40:25.288 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.allow_fallocate: 1 2024-03-21T12:40:25.288 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.allow_mmap_reads: 0 2024-03-21T12:40:25.288 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.allow_mmap_writes: 0 2024-03-21T12:40:25.288 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.use_direct_reads: 0 2024-03-21T12:40:25.289 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-03-21T12:40:25.289 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.create_missing_column_families: 0 2024-03-21T12:40:25.289 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.db_log_dir: 2024-03-21T12:40:25.289 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.wal_dir: 2024-03-21T12:40:25.289 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.table_cache_numshardbits: 6 2024-03-21T12:40:25.289 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.WAL_ttl_seconds: 0 2024-03-21T12:40:25.289 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.WAL_size_limit_MB: 0 2024-03-21T12:40:25.289 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-03-21T12:40:25.289 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-03-21T12:40:25.289 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.is_fd_close_on_exec: 1 2024-03-21T12:40:25.289 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.advise_random_on_open: 1 2024-03-21T12:40:25.289 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.db_write_buffer_size: 0 2024-03-21T12:40:25.289 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.write_buffer_manager: 0x55f03118a1e0 2024-03-21T12:40:25.289 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-03-21T12:40:25.289 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-03-21T12:40:25.290 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.use_adaptive_mutex: 0 2024-03-21T12:40:25.290 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.rate_limiter: (nil) 2024-03-21T12:40:25.290 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-03-21T12:40:25.290 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.wal_recovery_mode: 2 2024-03-21T12:40:25.290 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.enable_thread_tracking: 0 2024-03-21T12:40:25.290 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.enable_pipelined_write: 0 2024-03-21T12:40:25.290 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.unordered_write: 0 2024-03-21T12:40:25.290 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-03-21T12:40:25.290 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-03-21T12:40:25.290 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-03-21T12:40:25.290 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-03-21T12:40:25.291 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.row_cache: None 2024-03-21T12:40:25.291 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.wal_filter: None 2024-03-21T12:40:25.291 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-03-21T12:40:25.292 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.allow_ingest_behind: 0 2024-03-21T12:40:25.292 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.two_write_queues: 0 2024-03-21T12:40:25.292 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.manual_wal_flush: 0 2024-03-21T12:40:25.292 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.wal_compression: 0 2024-03-21T12:40:25.292 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.atomic_flush: 0 2024-03-21T12:40:25.292 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-03-21T12:40:25.292 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.persist_stats_to_disk: 0 2024-03-21T12:40:25.292 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.write_dbid_to_manifest: 0 2024-03-21T12:40:25.292 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.log_readahead_size: 0 2024-03-21T12:40:25.292 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-03-21T12:40:25.292 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.best_efforts_recovery: 0 2024-03-21T12:40:25.292 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-03-21T12:40:25.293 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-03-21T12:40:25.293 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.allow_data_in_errors: 0 2024-03-21T12:40:25.293 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.db_host_id: __hostname__ 2024-03-21T12:40:25.294 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.enforce_single_del_contracts: true 2024-03-21T12:40:25.294 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.max_background_jobs: 2 2024-03-21T12:40:25.294 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.max_background_compactions: -1 2024-03-21T12:40:25.294 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.max_subcompactions: 1 2024-03-21T12:40:25.294 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-03-21T12:40:25.294 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-03-21T12:40:25.294 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.delayed_write_rate : 16777216 2024-03-21T12:40:25.294 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.max_total_wal_size: 0 2024-03-21T12:40:25.295 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-03-21T12:40:25.295 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.stats_dump_period_sec: 600 2024-03-21T12:40:25.295 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.stats_persist_period_sec: 600 2024-03-21T12:40:25.295 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-03-21T12:40:25.295 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.max_open_files: -1 2024-03-21T12:40:25.295 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.bytes_per_sync: 0 2024-03-21T12:40:25.295 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.wal_bytes_per_sync: 0 2024-03-21T12:40:25.295 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.strict_bytes_per_sync: 0 2024-03-21T12:40:25.295 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.compaction_readahead_size: 0 2024-03-21T12:40:25.295 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.max_background_flushes: -1 2024-03-21T12:40:25.295 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Compression algorithms supported: 2024-03-21T12:40:25.295 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-03-21T12:40:25.295 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: kZSTD supported: 0 2024-03-21T12:40:25.295 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: kXpressCompression supported: 0 2024-03-21T12:40:25.295 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: kLZ4HCCompression supported: 1 2024-03-21T12:40:25.296 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: kZlibCompression supported: 1 2024-03-21T12:40:25.296 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: kSnappyCompression supported: 1 2024-03-21T12:40:25.296 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: kLZ4Compression supported: 1 2024-03-21T12:40:25.296 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: kBZip2Compression supported: 0 2024-03-21T12:40:25.296 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Fast CRC32 supported: Supported on x86 2024-03-21T12:40:25.296 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: DMutex implementation: pthread_mutex_t 2024-03-21T12:40:25.296 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi189/store.db/MANIFEST-000009 2024-03-21T12:40:25.296 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-03-21T12:40:25.296 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-03-21T12:40:25.296 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.merge_operator: 2024-03-21T12:40:25.296 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.compaction_filter: None 2024-03-21T12:40:25.296 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.compaction_filter_factory: None 2024-03-21T12:40:25.296 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.sst_partitioner_factory: None 2024-03-21T12:40:25.296 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.memtable_factory: SkipListFactory 2024-03-21T12:40:25.296 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.table_factory: BlockBasedTable 2024-03-21T12:40:25.296 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55f0311b98e0) 2024-03-21T12:40:25.297 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: cache_index_and_filter_blocks: 1 2024-03-21T12:40:25.297 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-03-21T12:40:25.297 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-03-21T12:40:25.297 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: pin_top_level_index_and_filter: 1 2024-03-21T12:40:25.297 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: index_type: 0 2024-03-21T12:40:25.297 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: data_block_index_type: 0 2024-03-21T12:40:25.297 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: index_shortening: 1 2024-03-21T12:40:25.297 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: data_block_hash_table_util_ratio: 0.750000 2024-03-21T12:40:25.297 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: checksum: 4 2024-03-21T12:40:25.297 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: no_block_cache: 0 2024-03-21T12:40:25.297 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: block_cache: 0x55f0311bd090 2024-03-21T12:40:25.297 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: block_cache_name: BinnedLRUCache 2024-03-21T12:40:25.297 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: block_cache_options: 2024-03-21T12:40:25.297 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: capacity : 536870912 2024-03-21T12:40:25.297 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: num_shard_bits : 4 2024-03-21T12:40:25.297 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: strict_capacity_limit : 0 2024-03-21T12:40:25.298 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: high_pri_pool_ratio: 0.000 2024-03-21T12:40:25.298 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: block_cache_compressed: (nil) 2024-03-21T12:40:25.298 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: persistent_cache: (nil) 2024-03-21T12:40:25.298 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: block_size: 4096 2024-03-21T12:40:25.298 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: block_size_deviation: 10 2024-03-21T12:40:25.298 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: block_restart_interval: 16 2024-03-21T12:40:25.298 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: index_block_restart_interval: 1 2024-03-21T12:40:25.298 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: metadata_block_size: 4096 2024-03-21T12:40:25.298 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: partition_filters: 0 2024-03-21T12:40:25.298 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: use_delta_encoding: 1 2024-03-21T12:40:25.298 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: filter_policy: bloomfilter 2024-03-21T12:40:25.298 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: whole_key_filtering: 1 2024-03-21T12:40:25.298 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: verify_compression: 0 2024-03-21T12:40:25.298 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: read_amp_bytes_per_bit: 0 2024-03-21T12:40:25.298 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: format_version: 5 2024-03-21T12:40:25.299 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: enable_index_compression: 1 2024-03-21T12:40:25.299 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: block_align: 0 2024-03-21T12:40:25.299 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: max_auto_readahead_size: 262144 2024-03-21T12:40:25.299 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: prepopulate_block_cache: 0 2024-03-21T12:40:25.299 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: initial_auto_readahead_size: 8192 2024-03-21T12:40:25.299 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: num_file_reads_for_auto_readahead: 2 2024-03-21T12:40:25.299 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.write_buffer_size: 33554432 2024-03-21T12:40:25.299 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.max_write_buffer_number: 2 2024-03-21T12:40:25.299 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.compression: NoCompression 2024-03-21T12:40:25.299 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.bottommost_compression: Disabled 2024-03-21T12:40:25.299 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.prefix_extractor: nullptr 2024-03-21T12:40:25.299 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-03-21T12:40:25.299 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.num_levels: 7 2024-03-21T12:40:25.299 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-03-21T12:40:25.299 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-03-21T12:40:25.300 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-03-21T12:40:25.300 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-03-21T12:40:25.300 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-03-21T12:40:25.300 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-03-21T12:40:25.300 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-03-21T12:40:25.300 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-03-21T12:40:25.300 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-03-21T12:40:25.300 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-03-21T12:40:25.300 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-03-21T12:40:25.300 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-03-21T12:40:25.300 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.compression_opts.window_bits: -14 2024-03-21T12:40:25.300 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.compression_opts.level: 32767 2024-03-21T12:40:25.300 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.compression_opts.strategy: 0 2024-03-21T12:40:25.300 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-03-21T12:40:25.300 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-03-21T12:40:25.301 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-03-21T12:40:25.301 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-03-21T12:40:25.301 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.compression_opts.enabled: false 2024-03-21T12:40:25.301 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-03-21T12:40:25.301 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-03-21T12:40:25.301 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-03-21T12:40:25.301 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-03-21T12:40:25.301 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.target_file_size_base: 67108864 2024-03-21T12:40:25.301 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.target_file_size_multiplier: 1 2024-03-21T12:40:25.301 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-03-21T12:40:25.302 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-03-21T12:40:25.302 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-03-21T12:40:25.302 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-03-21T12:40:25.302 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-03-21T12:40:25.302 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-03-21T12:40:25.302 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-03-21T12:40:25.302 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-03-21T12:40:25.302 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-03-21T12:40:25.302 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-03-21T12:40:25.302 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-03-21T12:40:25.302 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-03-21T12:40:25.302 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-03-21T12:40:25.302 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.arena_block_size: 1048576 2024-03-21T12:40:25.302 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-03-21T12:40:25.302 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-03-21T12:40:25.303 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.disable_auto_compactions: 0 2024-03-21T12:40:25.303 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-03-21T12:40:25.303 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-03-21T12:40:25.303 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-03-21T12:40:25.303 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-03-21T12:40:25.303 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-03-21T12:40:25.303 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-03-21T12:40:25.303 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-03-21T12:40:25.303 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-03-21T12:40:25.303 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-03-21T12:40:25.303 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-03-21T12:40:25.303 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.table_properties_collectors: 2024-03-21T12:40:25.303 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.inplace_update_support: 0 2024-03-21T12:40:25.303 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.inplace_update_num_locks: 10000 2024-03-21T12:40:25.303 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-03-21T12:40:25.303 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-03-21T12:40:25.304 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.memtable_huge_page_size: 0 2024-03-21T12:40:25.304 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.bloom_locality: 0 2024-03-21T12:40:25.304 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.max_successive_merges: 0 2024-03-21T12:40:25.304 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.optimize_filters_for_hits: 0 2024-03-21T12:40:25.304 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.paranoid_file_checks: 0 2024-03-21T12:40:25.304 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.force_consistency_checks: 1 2024-03-21T12:40:25.304 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.report_bg_io_stats: 0 2024-03-21T12:40:25.304 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.ttl: 2592000 2024-03-21T12:40:25.304 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.periodic_compaction_seconds: 0 2024-03-21T12:40:25.304 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-03-21T12:40:25.304 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-03-21T12:40:25.304 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.enable_blob_files: false 2024-03-21T12:40:25.304 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.min_blob_size: 0 2024-03-21T12:40:25.304 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.blob_file_size: 268435456 2024-03-21T12:40:25.304 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.blob_compression_type: NoCompression 2024-03-21T12:40:25.304 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.enable_blob_garbage_collection: false 2024-03-21T12:40:25.305 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-03-21T12:40:25.305 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-03-21T12:40:25.305 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-03-21T12:40:25.305 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.blob_file_starting_level: 0 2024-03-21T12:40:25.305 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-03-21T12:40:25.305 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 24.sst 2024-03-21T12:40:25.305 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-03-21T12:40:25.305 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi189/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 26, last_sequence is 12224, log_number is 22,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-03-21T12:40:25.305 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 22 2024-03-21T12:40:25.305 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 6a52e43b-5674-4439-844a-04d8929cd1f2 2024-03-21T12:40:25.305 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: EVENT_LOG_v1 {"time_micros": 1711024824923745, "job": 1, "event": "recovery_started", "wal_files": [22]} 2024-03-21T12:40:25.305 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #22 mode 2 2024-03-21T12:40:25.305 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: EVENT_LOG_v1 {"time_micros": 1711024824942556, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 27, "file_size": 4422485, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12221, "largest_seqno": 13979, "table_properties": {"data_size": 4414791, "index_size": 4883, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1797, "raw_key_size": 16078, "raw_average_key_size": 23, "raw_value_size": 4399733, "raw_average_value_size": 6321, "num_data_blocks": 221, "num_entries": 696, "num_filter_entries": 696, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1711024824, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "6a52e43b-5674-4439-844a-04d8929cd1f2", "db_session_id": "EIS9PBMYDIWINWR17BXW", "orig_file_number": 27, "seqno_to_time_mapping": "N/A"}} 2024-03-21T12:40:25.305 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: EVENT_LOG_v1 {"time_micros": 1711024824942668, "job": 1, "event": "recovery_finished"} 2024-03-21T12:40:25.305 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: [db/version_set.cc:5047] Creating manifest 29 2024-03-21T12:40:25.306 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-03-21T12:40:25.306 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-smithi189/store.db/000022.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-03-21T12:40:25.306 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55f0312aa000 2024-03-21T12:40:25.306 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: DB pointer 0x55f031296000 2024-03-21T12:40:25.306 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-03-21T12:40:25.306 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-03-21T12:40:25.306 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: ** DB Stats ** 2024-03-21T12:40:25.306 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-03-21T12:40:25.306 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-03-21T12:40:25.306 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-03-21T12:40:25.306 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-03-21T12:40:25.306 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-03-21T12:40:25.306 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-03-21T12:40:25.306 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-03-21T12:40:25.306 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: 2024-03-21T12:40:25.307 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: ** Compaction Stats [default] ** 2024-03-21T12:40:25.307 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-03-21T12:40:25.307 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-03-21T12:40:25.307 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: L0 1/0 4.22 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 511.5 0.01 0.00 1 0.008 0 0 0.0 0.0 2024-03-21T12:40:25.307 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: L6 1/0 6.53 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-03-21T12:40:25.307 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: Sum 2/0 10.74 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 511.5 0.01 0.00 1 0.008 0 0 0.0 0.0 2024-03-21T12:40:25.307 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 511.5 0.01 0.00 1 0.008 0 0 0.0 0.0 2024-03-21T12:40:25.307 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: 2024-03-21T12:40:25.307 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: ** Compaction Stats [default] ** 2024-03-21T12:40:25.307 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-03-21T12:40:25.307 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-03-21T12:40:25.307 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 511.5 0.01 0.00 1 0.008 0 0 0.0 0.0 2024-03-21T12:40:25.307 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: 2024-03-21T12:40:25.307 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-03-21T12:40:25.307 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: 2024-03-21T12:40:25.307 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-03-21T12:40:25.308 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: Flush(GB): cumulative 0.004, interval 0.004 2024-03-21T12:40:25.308 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-03-21T12:40:25.308 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-03-21T12:40:25.308 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-03-21T12:40:25.308 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: AddFile(Keys): cumulative 0, interval 0 2024-03-21T12:40:25.308 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: Cumulative compaction: 0.00 GB write, 175.76 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-21T12:40:25.308 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: Interval compaction: 0.00 GB write, 175.76 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-21T12:40:25.308 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-03-21T12:40:25.308 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: Block cache BinnedLRUCache@0x55f0311bd090#2 capacity: 512.00 MB usage: 7.11 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2024-03-21T12:40:25.308 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,2.03 KB,0.00038743%) IndexBlock(1,5.08 KB,0.000968575%) Misc(1,0.00 KB,0%) 2024-03-21T12:40:25.309 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: 2024-03-21T12:40:25.309 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: ** File Read Latency Histogram By Level [default] ** 2024-03-21T12:40:25.309 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: starting mon.smithi189 rank 1 at public addrs [v2:172.21.15.189:3300/0,v1:172.21.15.189:6789/0] at bind addrs [v2:172.21.15.189:3300/0,v1:172.21.15.189:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi189 fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 2024-03-21T12:40:25.309 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: mon.smithi189@-1(???) e2 preinit fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 2024-03-21T12:40:25.309 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: mon.smithi189@-1(???).mds e7 new map 2024-03-21T12:40:25.309 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: mon.smithi189@-1(???).mds e7 print_map 2024-03-21T12:40:25.309 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: e7 2024-03-21T12:40:25.309 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-03-21T12:40:25.309 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-03-21T12:40:25.309 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: legacy client fscid: 1 2024-03-21T12:40:25.309 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: 2024-03-21T12:40:25.309 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: Filesystem 'foofs' (1) 2024-03-21T12:40:25.309 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: fs_name foofs 2024-03-21T12:40:25.309 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: epoch 5 2024-03-21T12:40:25.310 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-03-21T12:40:25.310 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: created 2024-03-21T12:34:22.999525+0000 2024-03-21T12:40:25.310 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: modified 2024-03-21T12:34:29.019384+0000 2024-03-21T12:40:25.310 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: tableserver 0 2024-03-21T12:40:25.310 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: root 0 2024-03-21T12:40:25.310 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: session_timeout 60 2024-03-21T12:40:25.310 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: session_autoclose 300 2024-03-21T12:40:25.310 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: max_file_size 1099511627776 2024-03-21T12:40:25.310 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: max_xattr_size 65536 2024-03-21T12:40:25.310 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: required_client_features {} 2024-03-21T12:40:25.310 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: last_failure 0 2024-03-21T12:40:25.310 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: last_failure_osd_epoch 0 2024-03-21T12:40:25.310 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-03-21T12:40:25.310 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: max_mds 1 2024-03-21T12:40:25.310 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: in 0 2024-03-21T12:40:25.310 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: up {0=14488} 2024-03-21T12:40:25.311 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: failed 2024-03-21T12:40:25.311 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: damaged 2024-03-21T12:40:25.311 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: stopped 2024-03-21T12:40:25.311 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: data_pools [3] 2024-03-21T12:40:25.311 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: metadata_pool 2 2024-03-21T12:40:25.311 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: inline_data disabled 2024-03-21T12:40:25.311 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: balancer 2024-03-21T12:40:25.311 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: bal_rank_mask -1 2024-03-21T12:40:25.311 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: standby_count_wanted 1 2024-03-21T12:40:25.311 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: qdb_cluster leader: 0 members: 2024-03-21T12:40:25.311 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: [mds.foofs.smithi022.bcnxho{0:14488} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.22:6834/2456616276,v1:172.21.15.22:6835/2456616276] compat {c=[1],r=[1],i=[7ff]}] 2024-03-21T12:40:25.311 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: 2024-03-21T12:40:25.311 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: 2024-03-21T12:40:25.311 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: Standby daemons: 2024-03-21T12:40:25.312 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: 2024-03-21T12:40:25.312 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout: [mds.foofs.smithi189.lluvko{-1:24289} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.189:6832/2684579462,v1:172.21.15.189:6833/2684579462] compat {c=[1],r=[1],i=[7ff]}] 2024-03-21T12:40:25.312 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: mon.smithi189@-1(???).osd e73 crush map has features 3314933000854323200, adjusting msgr requires 2024-03-21T12:40:25.312 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: mon.smithi189@-1(???).osd e73 crush map has features 432629239337189376, adjusting msgr requires 2024-03-21T12:40:25.312 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: mon.smithi189@-1(???).osd e73 crush map has features 432629239337189376, adjusting msgr requires 2024-03-21T12:40:25.312 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: mon.smithi189@-1(???).osd e73 crush map has features 432629239337189376, adjusting msgr requires 2024-03-21T12:40:25.312 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 ceph-mon[53160]: mon.smithi189@-1(???).paxosservice(auth 1..30) refresh upgraded, format 0 -> 3 2024-03-21T12:40:25.312 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 bash[53141]: b353af3d8f3b6abef504d9964905f2616d9f4e964d160c9c7924d0d1ff28e456 2024-03-21T12:40:25.312 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:24 smithi189 systemd[1]: Started Ceph mon.smithi189 for 3fc7f0f2-e77e-11ee-95cd-87774f69a715. 2024-03-21T12:40:25.858 INFO:teuthology.orchestra.run.smithi022.stdout: 5 163669 11.83 MB/sec execute 205 sec latency 49.287 ms 2024-03-21T12:40:26.369 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:26 smithi022 ceph-mon[67047]: pgmap v12: 97 pgs: 97 active+clean; 335 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 32 KiB/s rd, 8.0 MiB/s wr, 793 op/s 2024-03-21T12:40:26.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:26 smithi189 ceph-mon[53160]: pgmap v12: 97 pgs: 97 active+clean; 335 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 32 KiB/s rd, 8.0 MiB/s wr, 793 op/s 2024-03-21T12:40:26.857 INFO:teuthology.orchestra.run.smithi022.stdout: 5 164103 11.83 MB/sec execute 206 sec latency 44.934 ms 2024-03-21T12:40:27.287 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:27 smithi189 ceph-mon[53160]: pgmap v14: 97 pgs: 97 active+clean; 335 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 5.2 MiB/s wr, 523 op/s 2024-03-21T12:40:27.287 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:27 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mon metadata", "id": "smithi022"}]: dispatch 2024-03-21T12:40:27.287 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:27 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mon metadata", "id": "smithi189"}]: dispatch 2024-03-21T12:40:27.287 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:27 smithi189 ceph-mon[53160]: mon.smithi189 calling monitor election 2024-03-21T12:40:27.287 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:27 smithi189 ceph-mon[53160]: mon.smithi022 calling monitor election 2024-03-21T12:40:27.287 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:27 smithi189 ceph-mon[53160]: mon.smithi022 is new leader, mons smithi022,smithi189 in quorum (ranks 0,1) 2024-03-21T12:40:27.287 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:27 smithi189 ceph-mon[53160]: monmap e3: 2 mons at {smithi022=[v2:172.21.15.22:3300/0,v1:172.21.15.22:6789/0],smithi189=[v2:172.21.15.189:3300/0,v1:172.21.15.189:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-03-21T12:40:27.287 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:27 smithi189 ceph-mon[53160]: fsmap foofs:1 {0=foofs.smithi022.bcnxho=up:active} 1 up:standby 2024-03-21T12:40:27.287 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:27 smithi189 ceph-mon[53160]: osdmap e73: 8 total, 8 up, 8 in 2024-03-21T12:40:27.287 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:27 smithi189 ceph-mon[53160]: mgrmap e41: smithi022.fektzs(active, since 20s), standbys: smithi189.cnxzvu 2024-03-21T12:40:27.287 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:27 smithi189 ceph-mon[53160]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-03-21T12:40:27.288 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:27 smithi189 ceph-mon[53160]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-21T12:40:27.288 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:27 smithi189 ceph-mon[53160]: daemon grafana.smithi022 on smithi022 is in error state 2024-03-21T12:40:27.288 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:27 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:27.288 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:27 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:27.288 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:27 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:40:27.385 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:27 smithi022 ceph-mon[67047]: pgmap v14: 97 pgs: 97 active+clean; 335 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 5.2 MiB/s wr, 523 op/s 2024-03-21T12:40:27.385 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:27 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mon metadata", "id": "smithi022"}]: dispatch 2024-03-21T12:40:27.385 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:27 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mon metadata", "id": "smithi189"}]: dispatch 2024-03-21T12:40:27.385 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:27 smithi022 ceph-mon[67047]: mon.smithi189 calling monitor election 2024-03-21T12:40:27.385 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:27 smithi022 ceph-mon[67047]: mon.smithi022 calling monitor election 2024-03-21T12:40:27.385 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:27 smithi022 ceph-mon[67047]: mon.smithi022 is new leader, mons smithi022,smithi189 in quorum (ranks 0,1) 2024-03-21T12:40:27.385 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:27 smithi022 ceph-mon[67047]: monmap e3: 2 mons at {smithi022=[v2:172.21.15.22:3300/0,v1:172.21.15.22:6789/0],smithi189=[v2:172.21.15.189:3300/0,v1:172.21.15.189:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-03-21T12:40:27.385 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:27 smithi022 ceph-mon[67047]: fsmap foofs:1 {0=foofs.smithi022.bcnxho=up:active} 1 up:standby 2024-03-21T12:40:27.385 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:27 smithi022 ceph-mon[67047]: osdmap e73: 8 total, 8 up, 8 in 2024-03-21T12:40:27.385 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:27 smithi022 ceph-mon[67047]: mgrmap e41: smithi022.fektzs(active, since 20s), standbys: smithi189.cnxzvu 2024-03-21T12:40:27.385 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:27 smithi022 ceph-mon[67047]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-03-21T12:40:27.385 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:27 smithi022 ceph-mon[67047]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-21T12:40:27.386 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:27 smithi022 ceph-mon[67047]: daemon grafana.smithi022 on smithi022 is in error state 2024-03-21T12:40:27.386 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:27 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:27.386 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:27 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:27.386 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:27 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:40:27.857 INFO:teuthology.orchestra.run.smithi022.stdout: 5 164455 11.84 MB/sec execute 207 sec latency 42.304 ms 2024-03-21T12:40:28.857 INFO:teuthology.orchestra.run.smithi022.stdout: 5 164845 11.82 MB/sec execute 208 sec latency 44.946 ms 2024-03-21T12:40:29.450 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:29 smithi022 ceph-mon[67047]: pgmap v15: 97 pgs: 97 active+clean; 338 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.6 MiB/s wr, 464 op/s 2024-03-21T12:40:29.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:29 smithi189 ceph-mon[53160]: pgmap v15: 97 pgs: 97 active+clean; 338 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.6 MiB/s wr, 464 op/s 2024-03-21T12:40:29.857 INFO:teuthology.orchestra.run.smithi022.stdout: 5 165271 11.83 MB/sec execute 209 sec latency 48.304 ms 2024-03-21T12:40:30.857 INFO:teuthology.orchestra.run.smithi022.stdout: 5 165595 11.82 MB/sec execute 210 sec latency 40.879 ms 2024-03-21T12:40:31.220 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:30 smithi022 ceph-mon[67047]: pgmap v16: 97 pgs: 97 active+clean; 338 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.0 MiB/s wr, 401 op/s 2024-03-21T12:40:31.221 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:30 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:31.221 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:30 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:31.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:30 smithi189 ceph-mon[53160]: pgmap v16: 97 pgs: 97 active+clean; 338 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.0 MiB/s wr, 401 op/s 2024-03-21T12:40:31.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:30 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:31.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:30 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:31.857 INFO:teuthology.orchestra.run.smithi022.stdout: 5 166041 11.79 MB/sec execute 211 sec latency 49.037 ms 2024-03-21T12:40:32.817 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:32 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:32.817 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:32 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:32.818 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:32 smithi189 ceph-mon[53160]: pgmap v17: 97 pgs: 97 active+clean; 337 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.4 MiB/s wr, 651 op/s 2024-03-21T12:40:32.857 INFO:teuthology.orchestra.run.smithi022.stdout: 5 166418 11.76 MB/sec execute 212 sec latency 38.668 ms 2024-03-21T12:40:33.011 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:32 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:33.012 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:32 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:33.012 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:32 smithi022 ceph-mon[67047]: pgmap v17: 97 pgs: 97 active+clean; 337 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.4 MiB/s wr, 651 op/s 2024-03-21T12:40:33.857 INFO:teuthology.orchestra.run.smithi022.stdout: 5 166677 11.75 MB/sec execute 213 sec latency 41.368 ms 2024-03-21T12:40:34.807 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:34 smithi022 ceph-mon[67047]: pgmap v18: 97 pgs: 97 active+clean; 337 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.5 MiB/s wr, 453 op/s 2024-03-21T12:40:34.859 INFO:teuthology.orchestra.run.smithi022.stdout: 5 167008 11.75 MB/sec execute 214 sec latency 65.380 ms 2024-03-21T12:40:35.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:34 smithi189 ceph-mon[53160]: pgmap v18: 97 pgs: 97 active+clean; 337 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.5 MiB/s wr, 453 op/s 2024-03-21T12:40:35.857 INFO:teuthology.orchestra.run.smithi022.stdout: 5 167391 11.74 MB/sec execute 215 sec latency 57.678 ms 2024-03-21T12:40:36.602 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:36 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:36.602 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:36 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:36.602 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:36 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:40:36.602 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:36 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:40:36.602 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:36 smithi022 ceph-mon[67047]: pgmap v19: 97 pgs: 97 active+clean; 337 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.6 MiB/s wr, 468 op/s 2024-03-21T12:40:36.602 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:36 smithi022 ceph-mon[67047]: pgmap v20: 97 pgs: 97 active+clean; 337 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.6 MiB/s wr, 566 op/s 2024-03-21T12:40:36.602 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:36 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:36.602 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:36 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:36.602 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:36 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:40:36.602 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:36 smithi022 ceph-mon[67047]: Reconfiguring mon.smithi022 (monmap changed)... 2024-03-21T12:40:36.602 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:36 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-21T12:40:36.602 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:36 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-21T12:40:36.603 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:36 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:40:36.603 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:36 smithi022 ceph-mon[67047]: Reconfiguring daemon mon.smithi022 on smithi022 2024-03-21T12:40:36.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:36 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:36.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:36 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:36.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:36 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:40:36.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:36 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:40:36.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:36 smithi189 ceph-mon[53160]: pgmap v19: 97 pgs: 97 active+clean; 337 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.6 MiB/s wr, 468 op/s 2024-03-21T12:40:36.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:36 smithi189 ceph-mon[53160]: pgmap v20: 97 pgs: 97 active+clean; 337 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.6 MiB/s wr, 566 op/s 2024-03-21T12:40:36.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:36 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:36.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:36 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:36.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:36 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:40:36.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:36 smithi189 ceph-mon[53160]: Reconfiguring mon.smithi022 (monmap changed)... 2024-03-21T12:40:36.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:36 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-21T12:40:36.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:36 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-21T12:40:36.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:36 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:40:36.787 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:36 smithi189 ceph-mon[53160]: Reconfiguring daemon mon.smithi022 on smithi022 2024-03-21T12:40:36.857 INFO:teuthology.orchestra.run.smithi022.stdout: 5 167749 11.75 MB/sec execute 216 sec latency 37.779 ms 2024-03-21T12:40:37.857 INFO:teuthology.orchestra.run.smithi022.stdout: 5 167992 11.73 MB/sec execute 217 sec latency 43.271 ms 2024-03-21T12:40:38.857 INFO:teuthology.orchestra.run.smithi022.stdout: 5 168312 11.71 MB/sec execute 218 sec latency 47.643 ms 2024-03-21T12:40:38.859 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:38 smithi022 ceph-mon[67047]: pgmap v21: 97 pgs: 97 active+clean; 336 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.6 MiB/s wr, 564 op/s 2024-03-21T12:40:39.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:38 smithi189 ceph-mon[53160]: pgmap v21: 97 pgs: 97 active+clean; 336 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.6 MiB/s wr, 564 op/s 2024-03-21T12:40:39.857 INFO:teuthology.orchestra.run.smithi022.stdout: 5 168597 11.69 MB/sec execute 219 sec latency 75.074 ms 2024-03-21T12:40:39.872 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:39 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:39.873 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:39 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:39.873 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:39 smithi022 ceph-mon[67047]: Reconfiguring mgr.smithi022.fektzs (monmap changed)... 2024-03-21T12:40:39.873 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:39 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi022.fektzs", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-21T12:40:39.873 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:39 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-21T12:40:39.873 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:39 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:40:39.873 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:39 smithi022 ceph-mon[67047]: Reconfiguring daemon mgr.smithi022.fektzs on smithi022 2024-03-21T12:40:39.998 INFO:teuthology.orchestra.run.smithi022.stdout:true 2024-03-21T12:40:40.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:39 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:40.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:39 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:40.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:39 smithi189 ceph-mon[53160]: Reconfiguring mgr.smithi022.fektzs (monmap changed)... 2024-03-21T12:40:40.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:39 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi022.fektzs", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-21T12:40:40.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:39 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-21T12:40:40.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:39 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:40:40.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:39 smithi189 ceph-mon[53160]: Reconfiguring daemon mgr.smithi022.fektzs on smithi022 2024-03-21T12:40:40.857 INFO:teuthology.orchestra.run.smithi022.stdout: 5 168833 11.67 MB/sec execute 220 sec latency 54.800 ms 2024-03-21T12:40:40.904 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:40 smithi022 ceph-mon[67047]: pgmap v22: 97 pgs: 97 active+clean; 336 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.6 MiB/s wr, 564 op/s 2024-03-21T12:40:40.905 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:40 smithi022 ceph-mon[67047]: from='client.44101 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:40:40.929 INFO:teuthology.orchestra.run.smithi022.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-21T12:40:40.929 INFO:teuthology.orchestra.run.smithi022.stdout:alertmanager.smithi022 smithi022 *:9093,9094 running (3m) 25s ago 12m 25.7M - 0.23.0 ba2b418f427c 14eb2a3ca318 2024-03-21T12:40:40.929 INFO:teuthology.orchestra.run.smithi022.stdout:crash.smithi022 smithi022 running (12m) 25s ago 12m 6405k - 17.2.0 e1d6a67b021e 47deb70fd250 2024-03-21T12:40:40.930 INFO:teuthology.orchestra.run.smithi022.stdout:crash.smithi189 smithi189 running (11m) 10s ago 11m 6405k - 17.2.0 e1d6a67b021e 9a2e0e9b1809 2024-03-21T12:40:40.930 INFO:teuthology.orchestra.run.smithi022.stdout:grafana.smithi022 smithi022 *:3000 error 25s ago 11m - - 2024-03-21T12:40:40.930 INFO:teuthology.orchestra.run.smithi022.stdout:mds.foofs.smithi022.bcnxho smithi022 running (6m) 25s ago 6m 37.7M - 17.2.0 e1d6a67b021e db26b67b8ee7 2024-03-21T12:40:40.930 INFO:teuthology.orchestra.run.smithi022.stdout:mds.foofs.smithi189.lluvko smithi189 running (6m) 10s ago 6m 16.7M - 17.2.0 e1d6a67b021e cf153aff1387 2024-03-21T12:40:40.930 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.smithi022.fektzs smithi022 *:8443,9283,8765 running (2m) 25s ago 13m 478M - 19.0.0-2267-g960fbbe4 19e12293512f d57cebd306b3 2024-03-21T12:40:40.930 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.smithi189.cnxzvu smithi189 *:8443,9283,8765 running (68s) 10s ago 11m 448M - 19.0.0-2267-g960fbbe4 19e12293512f 41663955c56f 2024-03-21T12:40:40.930 INFO:teuthology.orchestra.run.smithi022.stdout:mon.smithi022 smithi022 running (47s) 25s ago 13m 32.2M 2048M 19.0.0-2267-g960fbbe4 19e12293512f c620c9071b2c 2024-03-21T12:40:40.930 INFO:teuthology.orchestra.run.smithi022.stdout:mon.smithi189 smithi189 running (16s) 10s ago 11m 24.5M 2048M 19.0.0-2267-g960fbbe4 19e12293512f b353af3d8f3b 2024-03-21T12:40:40.930 INFO:teuthology.orchestra.run.smithi022.stdout:nfs.foo.0.0.smithi022.ynhbbq smithi022 *:2049 running (5m) 25s ago 5m 258M - 4.0 e1d6a67b021e 33e48e64c60b 2024-03-21T12:40:40.930 INFO:teuthology.orchestra.run.smithi022.stdout:nfs.foo.1.0.smithi189.yzsqml smithi189 *:2049 running (5m) 10s ago 5m 83.3M - 4.0 e1d6a67b021e 48fab2cbdde2 2024-03-21T12:40:40.930 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.smithi022 smithi022 *:9100 running (2m) 25s ago 11m 20.0M - 1.3.1 1dbe0e931976 33287f803f4f 2024-03-21T12:40:40.930 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.smithi189 smithi189 *:9100 running (2m) 10s ago 11m 21.1M - 1.3.1 1dbe0e931976 b51d872e438a 2024-03-21T12:40:40.930 INFO:teuthology.orchestra.run.smithi022.stdout:osd.0 smithi022 running (10m) 25s ago 10m 562M 1937M 17.2.0 e1d6a67b021e 54e6631bd10f 2024-03-21T12:40:40.931 INFO:teuthology.orchestra.run.smithi022.stdout:osd.1 smithi022 running (10m) 25s ago 10m 473M 1937M 17.2.0 e1d6a67b021e 68165c5d09cb 2024-03-21T12:40:40.931 INFO:teuthology.orchestra.run.smithi022.stdout:osd.2 smithi022 running (9m) 25s ago 9m 349M 1937M 17.2.0 e1d6a67b021e fbec9bef8433 2024-03-21T12:40:40.931 INFO:teuthology.orchestra.run.smithi022.stdout:osd.3 smithi022 running (9m) 25s ago 9m 491M 1937M 17.2.0 e1d6a67b021e b1462ffc1b9e 2024-03-21T12:40:40.931 INFO:teuthology.orchestra.run.smithi022.stdout:osd.4 smithi189 running (8m) 10s ago 8m 532M 2704M 17.2.0 e1d6a67b021e 1bedc4eb02ed 2024-03-21T12:40:40.931 INFO:teuthology.orchestra.run.smithi022.stdout:osd.5 smithi189 running (8m) 10s ago 8m 340M 2704M 17.2.0 e1d6a67b021e 25fbab9786a5 2024-03-21T12:40:40.931 INFO:teuthology.orchestra.run.smithi022.stdout:osd.6 smithi189 running (8m) 10s ago 8m 525M 2704M 17.2.0 e1d6a67b021e 0c192fcfd65b 2024-03-21T12:40:40.931 INFO:teuthology.orchestra.run.smithi022.stdout:osd.7 smithi189 running (7m) 10s ago 7m 448M 2704M 17.2.0 e1d6a67b021e 3d0723e1f863 2024-03-21T12:40:40.931 INFO:teuthology.orchestra.run.smithi022.stdout:prometheus.smithi022 smithi022 *:9095 running (75s) 25s ago 10m 65.3M - 2.33.4 514e6a882f6e 8171bbe41d2c 2024-03-21T12:40:41.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:40 smithi189 ceph-mon[53160]: pgmap v22: 97 pgs: 97 active+clean; 336 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.6 MiB/s wr, 564 op/s 2024-03-21T12:40:41.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:40 smithi189 ceph-mon[53160]: from='client.44101 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:40:41.524 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-03-21T12:40:41.524 INFO:teuthology.orchestra.run.smithi022.stdout: "mon": { 2024-03-21T12:40:41.524 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 2 2024-03-21T12:40:41.524 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:40:41.524 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr": { 2024-03-21T12:40:41.524 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 2 2024-03-21T12:40:41.524 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:40:41.524 INFO:teuthology.orchestra.run.smithi022.stdout: "osd": { 2024-03-21T12:40:41.525 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-03-21T12:40:41.525 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:40:41.525 INFO:teuthology.orchestra.run.smithi022.stdout: "mds": { 2024-03-21T12:40:41.525 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-21T12:40:41.525 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:40:41.525 INFO:teuthology.orchestra.run.smithi022.stdout: "overall": { 2024-03-21T12:40:41.525 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-03-21T12:40:41.525 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 4 2024-03-21T12:40:41.525 INFO:teuthology.orchestra.run.smithi022.stdout: } 2024-03-21T12:40:41.525 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-03-21T12:40:41.857 INFO:teuthology.orchestra.run.smithi022.stdout: 5 169031 11.65 MB/sec execute 221 sec latency 59.555 ms 2024-03-21T12:40:42.039 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-03-21T12:40:42.040 INFO:teuthology.orchestra.run.smithi022.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93", 2024-03-21T12:40:42.040 INFO:teuthology.orchestra.run.smithi022.stdout: "in_progress": true, 2024-03-21T12:40:42.040 INFO:teuthology.orchestra.run.smithi022.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-21T12:40:42.040 INFO:teuthology.orchestra.run.smithi022.stdout: "services_complete": [ 2024-03-21T12:40:42.040 INFO:teuthology.orchestra.run.smithi022.stdout: "mon", 2024-03-21T12:40:42.040 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr" 2024-03-21T12:40:42.040 INFO:teuthology.orchestra.run.smithi022.stdout: ], 2024-03-21T12:40:42.040 INFO:teuthology.orchestra.run.smithi022.stdout: "progress": "4/23 daemons upgraded", 2024-03-21T12:40:42.040 INFO:teuthology.orchestra.run.smithi022.stdout: "message": "Currently upgrading mon daemons", 2024-03-21T12:40:42.040 INFO:teuthology.orchestra.run.smithi022.stdout: "is_paused": false 2024-03-21T12:40:42.040 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-03-21T12:40:42.189 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:41 smithi022 ceph-mon[67047]: from='client.34156 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:40:42.190 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:41 smithi022 ceph-mon[67047]: from='client.34160 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:40:42.190 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:41 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:42.190 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:41 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:42.190 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:41 smithi022 ceph-mon[67047]: Reconfiguring crash.smithi022 (monmap changed)... 2024-03-21T12:40:42.190 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:41 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi022", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-03-21T12:40:42.190 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:41 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:40:42.190 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:41 smithi022 ceph-mon[67047]: Reconfiguring daemon crash.smithi022 on smithi022 2024-03-21T12:40:42.190 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:41 smithi022 ceph-mon[67047]: from='client.? 172.21.15.22:0/1604660216' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:40:42.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:41 smithi189 ceph-mon[53160]: from='client.34156 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:40:42.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:41 smithi189 ceph-mon[53160]: from='client.34160 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:40:42.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:41 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:42.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:41 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:42.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:41 smithi189 ceph-mon[53160]: Reconfiguring crash.smithi022 (monmap changed)... 2024-03-21T12:40:42.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:41 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi022", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-03-21T12:40:42.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:41 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:40:42.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:41 smithi189 ceph-mon[53160]: Reconfiguring daemon crash.smithi022 on smithi022 2024-03-21T12:40:42.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:41 smithi189 ceph-mon[53160]: from='client.? 172.21.15.22:0/1604660216' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:40:42.640 INFO:teuthology.orchestra.run.smithi022.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-03-21T12:40:42.640 INFO:teuthology.orchestra.run.smithi022.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-21T12:40:42.640 INFO:teuthology.orchestra.run.smithi022.stdout: daemon grafana.smithi022 on smithi022 is in error state 2024-03-21T12:40:42.857 INFO:teuthology.orchestra.run.smithi022.stdout: 5 169320 11.63 MB/sec execute 222 sec latency 50.265 ms 2024-03-21T12:40:43.207 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:43 smithi022 ceph-mon[67047]: pgmap v23: 97 pgs: 97 active+clean; 336 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 3.7 KiB/s rd, 2.6 MiB/s wr, 252 op/s 2024-03-21T12:40:43.207 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:43 smithi022 ceph-mon[67047]: from='client.34168 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:40:43.208 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:43 smithi022 ceph-mon[67047]: from='client.? 172.21.15.22:0/3104867052' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-21T12:40:43.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:43 smithi189 ceph-mon[53160]: pgmap v23: 97 pgs: 97 active+clean; 336 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 3.7 KiB/s rd, 2.6 MiB/s wr, 252 op/s 2024-03-21T12:40:43.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:43 smithi189 ceph-mon[53160]: from='client.34168 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:40:43.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:43 smithi189 ceph-mon[53160]: from='client.? 172.21.15.22:0/3104867052' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-21T12:40:43.857 INFO:teuthology.orchestra.run.smithi022.stdout: 5 169683 11.58 MB/sec execute 223 sec latency 68.670 ms 2024-03-21T12:40:44.722 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:44 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:44.722 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:44 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:44.722 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:44 smithi022 ceph-mon[67047]: pgmap v24: 97 pgs: 97 active+clean; 336 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.1 MiB/s wr, 477 op/s 2024-03-21T12:40:44.722 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:44 smithi022 ceph-mon[67047]: Reconfiguring osd.0 (monmap changed)... 2024-03-21T12:40:44.722 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:44 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-03-21T12:40:44.722 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:44 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:40:44.722 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:44 smithi022 ceph-mon[67047]: Reconfiguring daemon osd.0 on smithi022 2024-03-21T12:40:44.857 INFO:teuthology.orchestra.run.smithi022.stdout: 5 169968 11.55 MB/sec execute 224 sec latency 48.670 ms 2024-03-21T12:40:45.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:44 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:45.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:44 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:45.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:44 smithi189 ceph-mon[53160]: pgmap v24: 97 pgs: 97 active+clean; 336 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.1 MiB/s wr, 477 op/s 2024-03-21T12:40:45.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:44 smithi189 ceph-mon[53160]: Reconfiguring osd.0 (monmap changed)... 2024-03-21T12:40:45.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:44 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-03-21T12:40:45.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:44 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:40:45.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:44 smithi189 ceph-mon[53160]: Reconfiguring daemon osd.0 on smithi022 2024-03-21T12:40:45.858 INFO:teuthology.orchestra.run.smithi022.stdout: 5 170226 11.55 MB/sec execute 225 sec latency 46.763 ms 2024-03-21T12:40:46.857 INFO:teuthology.orchestra.run.smithi022.stdout: 5 170527 11.54 MB/sec execute 226 sec latency 56.566 ms 2024-03-21T12:40:47.028 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:46 smithi022 ceph-mon[67047]: pgmap v25: 97 pgs: 97 active+clean; 336 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.9 MiB/s wr, 459 op/s 2024-03-21T12:40:47.029 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:46 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:47.029 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:46 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:47.029 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:46 smithi022 ceph-mon[67047]: Reconfiguring osd.1 (monmap changed)... 2024-03-21T12:40:47.029 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:46 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-03-21T12:40:47.029 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:46 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:40:47.029 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:46 smithi022 ceph-mon[67047]: Reconfiguring daemon osd.1 on smithi022 2024-03-21T12:40:47.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:46 smithi189 ceph-mon[53160]: pgmap v25: 97 pgs: 97 active+clean; 336 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.9 MiB/s wr, 459 op/s 2024-03-21T12:40:47.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:46 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:47.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:46 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:47.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:46 smithi189 ceph-mon[53160]: Reconfiguring osd.1 (monmap changed)... 2024-03-21T12:40:47.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:46 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-03-21T12:40:47.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:46 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:40:47.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:46 smithi189 ceph-mon[53160]: Reconfiguring daemon osd.1 on smithi022 2024-03-21T12:40:47.857 INFO:teuthology.orchestra.run.smithi022.stdout: 5 170892 11.54 MB/sec execute 227 sec latency 46.881 ms 2024-03-21T12:40:48.858 INFO:teuthology.orchestra.run.smithi022.stdout: 5 171236 11.53 MB/sec execute 228 sec latency 46.066 ms 2024-03-21T12:40:49.076 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:48 smithi022 ceph-mon[67047]: pgmap v26: 97 pgs: 97 active+clean; 332 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 5.6 MiB/s wr, 563 op/s 2024-03-21T12:40:49.077 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:48 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:49.077 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:48 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:49.077 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:48 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-03-21T12:40:49.077 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:48 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:40:49.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:48 smithi189 ceph-mon[53160]: pgmap v26: 97 pgs: 97 active+clean; 332 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 5.6 MiB/s wr, 563 op/s 2024-03-21T12:40:49.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:48 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:49.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:48 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:49.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:48 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-03-21T12:40:49.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:48 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:40:49.858 INFO:teuthology.orchestra.run.smithi022.stdout: 5 171488 11.53 MB/sec execute 229 sec latency 54.623 ms 2024-03-21T12:40:50.093 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:49 smithi022 ceph-mon[67047]: Reconfiguring osd.2 (monmap changed)... 2024-03-21T12:40:50.093 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:49 smithi022 ceph-mon[67047]: Reconfiguring daemon osd.2 on smithi022 2024-03-21T12:40:50.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:49 smithi189 ceph-mon[53160]: Reconfiguring osd.2 (monmap changed)... 2024-03-21T12:40:50.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:49 smithi189 ceph-mon[53160]: Reconfiguring daemon osd.2 on smithi022 2024-03-21T12:40:50.858 INFO:teuthology.orchestra.run.smithi022.stdout: 5 171680 11.49 MB/sec execute 230 sec latency 57.095 ms 2024-03-21T12:40:51.107 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:50 smithi022 ceph-mon[67047]: pgmap v27: 97 pgs: 97 active+clean; 332 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 3.6 MiB/s wr, 360 op/s 2024-03-21T12:40:51.108 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:50 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:40:51.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:50 smithi189 ceph-mon[53160]: pgmap v27: 97 pgs: 97 active+clean; 332 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 3.6 MiB/s wr, 360 op/s 2024-03-21T12:40:51.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:50 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:40:51.858 INFO:teuthology.orchestra.run.smithi022.stdout: 5 171949 11.48 MB/sec execute 231 sec latency 72.583 ms 2024-03-21T12:40:52.380 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:52 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:52.380 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:52 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:52.380 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:52 smithi022 ceph-mon[67047]: Reconfiguring osd.3 (monmap changed)... 2024-03-21T12:40:52.380 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:52 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-03-21T12:40:52.380 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:52 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:40:52.380 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:52 smithi022 ceph-mon[67047]: Reconfiguring daemon osd.3 on smithi022 2024-03-21T12:40:52.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:52 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:52.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:52 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:52.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:52 smithi189 ceph-mon[53160]: Reconfiguring osd.3 (monmap changed)... 2024-03-21T12:40:52.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:52 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-03-21T12:40:52.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:52 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:40:52.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:52 smithi189 ceph-mon[53160]: Reconfiguring daemon osd.3 on smithi022 2024-03-21T12:40:52.858 INFO:teuthology.orchestra.run.smithi022.stdout: 5 172291 11.47 MB/sec execute 232 sec latency 50.429 ms 2024-03-21T12:40:53.394 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:53 smithi022 ceph-mon[67047]: pgmap v28: 97 pgs: 97 active+clean; 332 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 3.6 MiB/s wr, 360 op/s 2024-03-21T12:40:53.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:53 smithi189 ceph-mon[53160]: pgmap v28: 97 pgs: 97 active+clean; 332 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 3.6 MiB/s wr, 360 op/s 2024-03-21T12:40:53.858 INFO:teuthology.orchestra.run.smithi022.stdout: 5 172588 11.46 MB/sec execute 233 sec latency 45.956 ms 2024-03-21T12:40:54.656 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:54 smithi022 ceph-mon[67047]: pgmap v29: 97 pgs: 97 active+clean; 344 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.5 MiB/s wr, 524 op/s 2024-03-21T12:40:54.656 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:54 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:54.656 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:54 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:54.656 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:54 smithi022 ceph-mon[67047]: Reconfiguring mds.foofs.smithi022.bcnxho (monmap changed)... 2024-03-21T12:40:54.656 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:54 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi022.bcnxho", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-21T12:40:54.656 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:54 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:40:54.656 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:54 smithi022 ceph-mon[67047]: Reconfiguring daemon mds.foofs.smithi022.bcnxho on smithi022 2024-03-21T12:40:54.858 INFO:teuthology.orchestra.run.smithi022.stdout: 5 172922 11.44 MB/sec execute 234 sec latency 47.240 ms 2024-03-21T12:40:55.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:54 smithi189 ceph-mon[53160]: pgmap v29: 97 pgs: 97 active+clean; 344 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.5 MiB/s wr, 524 op/s 2024-03-21T12:40:55.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:54 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:55.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:54 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:55.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:54 smithi189 ceph-mon[53160]: Reconfiguring mds.foofs.smithi022.bcnxho (monmap changed)... 2024-03-21T12:40:55.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:54 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi022.bcnxho", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-21T12:40:55.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:54 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:40:55.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:54 smithi189 ceph-mon[53160]: Reconfiguring daemon mds.foofs.smithi022.bcnxho on smithi022 2024-03-21T12:40:55.858 INFO:teuthology.orchestra.run.smithi022.stdout: 5 173346 11.40 MB/sec execute 235 sec latency 41.187 ms 2024-03-21T12:40:56.858 INFO:teuthology.orchestra.run.smithi022.stdout: 5 173603 11.38 MB/sec execute 236 sec latency 41.907 ms 2024-03-21T12:40:57.435 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:57 smithi022 ceph-mon[67047]: pgmap v30: 97 pgs: 97 active+clean; 344 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 3.5 MiB/s wr, 344 op/s 2024-03-21T12:40:57.436 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:57 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:57.436 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:57 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:57.436 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:57 smithi022 ceph-mon[67047]: Reconfiguring crash.smithi189 (monmap changed)... 2024-03-21T12:40:57.436 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:57 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi189", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-03-21T12:40:57.436 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:57 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:40:57.436 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:57 smithi022 ceph-mon[67047]: Reconfiguring daemon crash.smithi189 on smithi189 2024-03-21T12:40:57.455 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:57 smithi189 ceph-mon[53160]: pgmap v30: 97 pgs: 97 active+clean; 344 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 3.5 MiB/s wr, 344 op/s 2024-03-21T12:40:57.455 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:57 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:57.455 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:57 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:57.455 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:57 smithi189 ceph-mon[53160]: Reconfiguring crash.smithi189 (monmap changed)... 2024-03-21T12:40:57.455 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:57 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi189", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-03-21T12:40:57.456 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:57 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:40:57.456 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:57 smithi189 ceph-mon[53160]: Reconfiguring daemon crash.smithi189 on smithi189 2024-03-21T12:40:57.858 INFO:teuthology.orchestra.run.smithi022.stdout: 5 173924 11.39 MB/sec execute 237 sec latency 79.074 ms 2024-03-21T12:40:58.858 INFO:teuthology.orchestra.run.smithi022.stdout: 5 174336 11.39 MB/sec execute 238 sec latency 53.136 ms 2024-03-21T12:40:59.233 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:59 smithi022 ceph-mon[67047]: pgmap v31: 97 pgs: 97 active+clean; 334 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.3 MiB/s wr, 561 op/s 2024-03-21T12:40:59.233 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:59 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:59.233 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:59 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:59.233 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:59 smithi022 ceph-mon[67047]: Reconfiguring mgr.smithi189.cnxzvu (monmap changed)... 2024-03-21T12:40:59.233 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:59 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi189.cnxzvu", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-21T12:40:59.233 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:59 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-21T12:40:59.233 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:59 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:40:59.233 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:40:59 smithi022 ceph-mon[67047]: Reconfiguring daemon mgr.smithi189.cnxzvu on smithi189 2024-03-21T12:40:59.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:59 smithi189 ceph-mon[53160]: pgmap v31: 97 pgs: 97 active+clean; 334 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.3 MiB/s wr, 561 op/s 2024-03-21T12:40:59.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:59 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:59.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:59 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:40:59.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:59 smithi189 ceph-mon[53160]: Reconfiguring mgr.smithi189.cnxzvu (monmap changed)... 2024-03-21T12:40:59.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:59 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi189.cnxzvu", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-21T12:40:59.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:59 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-21T12:40:59.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:59 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:40:59.287 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:40:59 smithi189 ceph-mon[53160]: Reconfiguring daemon mgr.smithi189.cnxzvu on smithi189 2024-03-21T12:40:59.859 INFO:teuthology.orchestra.run.smithi022.stdout: 5 174727 11.39 MB/sec execute 239 sec latency 36.996 ms 2024-03-21T12:41:00.858 INFO:teuthology.orchestra.run.smithi022.stdout: 5 175025 11.40 MB/sec execute 240 sec latency 42.510 ms 2024-03-21T12:41:01.034 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:00 smithi022 ceph-mon[67047]: pgmap v32: 97 pgs: 97 active+clean; 334 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 3.7 MiB/s wr, 380 op/s 2024-03-21T12:41:01.035 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:00 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:41:01.035 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:00 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:41:01.035 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:00 smithi022 ceph-mon[67047]: Reconfiguring mon.smithi189 (monmap changed)... 2024-03-21T12:41:01.035 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:00 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-21T12:41:01.035 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:00 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-21T12:41:01.035 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:00 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:41:01.035 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:00 smithi022 ceph-mon[67047]: Reconfiguring daemon mon.smithi189 on smithi189 2024-03-21T12:41:01.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:00 smithi189 ceph-mon[53160]: pgmap v32: 97 pgs: 97 active+clean; 334 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 3.7 MiB/s wr, 380 op/s 2024-03-21T12:41:01.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:00 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:41:01.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:00 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:41:01.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:00 smithi189 ceph-mon[53160]: Reconfiguring mon.smithi189 (monmap changed)... 2024-03-21T12:41:01.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:00 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-21T12:41:01.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:00 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-21T12:41:01.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:00 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:41:01.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:00 smithi189 ceph-mon[53160]: Reconfiguring daemon mon.smithi189 on smithi189 2024-03-21T12:41:01.858 INFO:teuthology.orchestra.run.smithi022.stdout: 5 175306 11.38 MB/sec execute 241 sec latency 40.346 ms 2024-03-21T12:41:02.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:02 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:41:02.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:02 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:41:02.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:02 smithi189 ceph-mon[53160]: Reconfiguring osd.4 (monmap changed)... 2024-03-21T12:41:02.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:02 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-03-21T12:41:02.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:02 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:41:02.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:02 smithi189 ceph-mon[53160]: Reconfiguring daemon osd.4 on smithi189 2024-03-21T12:41:02.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:02 smithi189 ceph-mon[53160]: pgmap v33: 97 pgs: 97 active+clean; 334 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 3.7 MiB/s wr, 380 op/s 2024-03-21T12:41:02.809 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:02 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:41:02.809 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:02 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:41:02.809 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:02 smithi022 ceph-mon[67047]: Reconfiguring osd.4 (monmap changed)... 2024-03-21T12:41:02.809 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:02 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-03-21T12:41:02.809 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:02 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:41:02.809 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:02 smithi022 ceph-mon[67047]: Reconfiguring daemon osd.4 on smithi189 2024-03-21T12:41:02.809 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:02 smithi022 ceph-mon[67047]: pgmap v33: 97 pgs: 97 active+clean; 334 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 3.7 MiB/s wr, 380 op/s 2024-03-21T12:41:02.858 INFO:teuthology.orchestra.run.smithi022.stdout: 5 175757 11.38 MB/sec execute 242 sec latency 61.766 ms 2024-03-21T12:41:03.858 INFO:teuthology.orchestra.run.smithi022.stdout: 5 176202 11.39 MB/sec execute 243 sec latency 29.146 ms 2024-03-21T12:41:04.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:04 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:41:04.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:04 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:41:04.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:04 smithi022 ceph-mon[67047]: Reconfiguring osd.5 (monmap changed)... 2024-03-21T12:41:04.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:04 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-03-21T12:41:04.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:04 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:41:04.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:04 smithi022 ceph-mon[67047]: Reconfiguring daemon osd.5 on smithi189 2024-03-21T12:41:04.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:04 smithi022 ceph-mon[67047]: pgmap v34: 97 pgs: 97 active+clean; 343 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 6.0 MiB/s wr, 577 op/s 2024-03-21T12:41:04.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:04 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:41:04.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:04 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:41:04.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:04 smithi189 ceph-mon[53160]: Reconfiguring osd.5 (monmap changed)... 2024-03-21T12:41:04.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:04 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-03-21T12:41:04.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:04 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:41:04.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:04 smithi189 ceph-mon[53160]: Reconfiguring daemon osd.5 on smithi189 2024-03-21T12:41:04.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:04 smithi189 ceph-mon[53160]: pgmap v34: 97 pgs: 97 active+clean; 343 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 6.0 MiB/s wr, 577 op/s 2024-03-21T12:41:04.858 INFO:teuthology.orchestra.run.smithi022.stdout: 5 176729 11.37 MB/sec execute 244 sec latency 51.922 ms 2024-03-21T12:41:05.858 INFO:teuthology.orchestra.run.smithi022.stdout: 5 177121 11.35 MB/sec execute 245 sec latency 52.461 ms 2024-03-21T12:41:06.364 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:06 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:41:06.365 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:06 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:41:06.365 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:06 smithi022 ceph-mon[67047]: Reconfiguring osd.6 (monmap changed)... 2024-03-21T12:41:06.365 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:06 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-03-21T12:41:06.365 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:06 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:41:06.365 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:06 smithi022 ceph-mon[67047]: Reconfiguring daemon osd.6 on smithi189 2024-03-21T12:41:06.365 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:06 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:41:06.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:06 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:41:06.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:06 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:41:06.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:06 smithi189 ceph-mon[53160]: Reconfiguring osd.6 (monmap changed)... 2024-03-21T12:41:06.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:06 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-03-21T12:41:06.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:06 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:41:06.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:06 smithi189 ceph-mon[53160]: Reconfiguring daemon osd.6 on smithi189 2024-03-21T12:41:06.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:06 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:41:06.858 INFO:teuthology.orchestra.run.smithi022.stdout: 5 177555 11.37 MB/sec execute 246 sec latency 58.694 ms 2024-03-21T12:41:07.376 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:07 smithi022 ceph-mon[67047]: pgmap v35: 97 pgs: 97 active+clean; 343 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.1 MiB/s wr, 413 op/s 2024-03-21T12:41:07.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:07 smithi189 ceph-mon[53160]: pgmap v35: 97 pgs: 97 active+clean; 343 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.1 MiB/s wr, 413 op/s 2024-03-21T12:41:07.858 INFO:teuthology.orchestra.run.smithi022.stdout: 5 178113 11.39 MB/sec execute 247 sec latency 59.116 ms 2024-03-21T12:41:08.412 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:08 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:41:08.412 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:08 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:41:08.412 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:08 smithi022 ceph-mon[67047]: Reconfiguring osd.7 (monmap changed)... 2024-03-21T12:41:08.412 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:08 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-03-21T12:41:08.413 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:08 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:41:08.413 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:08 smithi022 ceph-mon[67047]: Reconfiguring daemon osd.7 on smithi189 2024-03-21T12:41:08.413 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:08 smithi022 ceph-mon[67047]: pgmap v36: 97 pgs: 97 active+clean; 343 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.6 MiB/s wr, 693 op/s 2024-03-21T12:41:08.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:08 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:41:08.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:08 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:41:08.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:08 smithi189 ceph-mon[53160]: Reconfiguring osd.7 (monmap changed)... 2024-03-21T12:41:08.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:08 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-03-21T12:41:08.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:08 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:41:08.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:08 smithi189 ceph-mon[53160]: Reconfiguring daemon osd.7 on smithi189 2024-03-21T12:41:08.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:08 smithi189 ceph-mon[53160]: pgmap v36: 97 pgs: 97 active+clean; 343 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.6 MiB/s wr, 693 op/s 2024-03-21T12:41:08.858 INFO:teuthology.orchestra.run.smithi022.stdout: 5 178539 11.41 MB/sec execute 248 sec latency 63.670 ms 2024-03-21T12:41:09.858 INFO:teuthology.orchestra.run.smithi022.stdout: 5 178846 11.40 MB/sec execute 249 sec latency 62.637 ms 2024-03-21T12:41:10.457 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:10 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:41:10.458 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:10 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:41:10.458 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:10 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi189.lluvko", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-21T12:41:10.458 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:10 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:41:10.510 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:10 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:41:10.511 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:10 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:41:10.511 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:10 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi189.lluvko", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-21T12:41:10.511 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:10 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:41:10.858 INFO:teuthology.orchestra.run.smithi022.stdout: 5 179288 11.39 MB/sec execute 250 sec latency 34.830 ms 2024-03-21T12:41:11.378 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:11 smithi022 ceph-mon[67047]: Reconfiguring mds.foofs.smithi189.lluvko (monmap changed)... 2024-03-21T12:41:11.378 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:11 smithi022 ceph-mon[67047]: Reconfiguring daemon mds.foofs.smithi189.lluvko on smithi189 2024-03-21T12:41:11.378 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:11 smithi022 ceph-mon[67047]: pgmap v37: 97 pgs: 97 active+clean; 343 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.8 MiB/s wr, 477 op/s 2024-03-21T12:41:11.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:11 smithi189 ceph-mon[53160]: Reconfiguring mds.foofs.smithi189.lluvko (monmap changed)... 2024-03-21T12:41:11.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:11 smithi189 ceph-mon[53160]: Reconfiguring daemon mds.foofs.smithi189.lluvko on smithi189 2024-03-21T12:41:11.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:11 smithi189 ceph-mon[53160]: pgmap v37: 97 pgs: 97 active+clean; 343 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.8 MiB/s wr, 477 op/s 2024-03-21T12:41:11.858 INFO:teuthology.orchestra.run.smithi022.stdout: 5 179605 11.38 MB/sec execute 251 sec latency 33.273 ms 2024-03-21T12:41:12.389 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:12 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:41:12.389 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:12 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:41:12.389 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:12 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:41:12.389 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:12 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:41:12.389 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:12 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:41:12.389 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:12 smithi022 ceph-mon[67047]: Upgrade: Setting container_image for all mon 2024-03-21T12:41:12.389 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:12 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:41:12.389 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:12 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi022"}]: dispatch 2024-03-21T12:41:12.389 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:12 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi022"}]': finished 2024-03-21T12:41:12.390 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:12 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi189"}]: dispatch 2024-03-21T12:41:12.650 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:12 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi189"}]': finished 2024-03-21T12:41:12.651 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:12 smithi022 ceph-mon[67047]: pgmap v38: 97 pgs: 97 active+clean; 343 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.8 MiB/s wr, 477 op/s 2024-03-21T12:41:12.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:12 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:41:12.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:12 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:41:12.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:12 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:41:12.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:12 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:41:12.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:12 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:41:12.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:12 smithi189 ceph-mon[53160]: Upgrade: Setting container_image for all mon 2024-03-21T12:41:12.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:12 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:41:12.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:12 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi022"}]: dispatch 2024-03-21T12:41:12.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:12 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi022"}]': finished 2024-03-21T12:41:12.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:12 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi189"}]: dispatch 2024-03-21T12:41:12.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:12 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi189"}]': finished 2024-03-21T12:41:12.787 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:12 smithi189 ceph-mon[53160]: pgmap v38: 97 pgs: 97 active+clean; 343 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.8 MiB/s wr, 477 op/s 2024-03-21T12:41:12.858 INFO:teuthology.orchestra.run.smithi022.stdout: 5 179963 11.38 MB/sec execute 252 sec latency 47.140 ms 2024-03-21T12:41:13.242 INFO:teuthology.orchestra.run.smithi022.stdout:true 2024-03-21T12:41:13.664 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:13 smithi022 ceph-mon[67047]: from='client.34176 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:41:13.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:13 smithi189 ceph-mon[53160]: from='client.34176 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:41:13.858 INFO:teuthology.orchestra.run.smithi022.stdout: 5 180394 11.35 MB/sec execute 253 sec latency 40.118 ms 2024-03-21T12:41:14.338 INFO:teuthology.orchestra.run.smithi022.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-21T12:41:14.338 INFO:teuthology.orchestra.run.smithi022.stdout:alertmanager.smithi022 smithi022 *:9093,9094 running (3m) 59s ago 12m 25.7M - 0.23.0 ba2b418f427c 14eb2a3ca318 2024-03-21T12:41:14.338 INFO:teuthology.orchestra.run.smithi022.stdout:crash.smithi022 smithi022 running (12m) 59s ago 12m 6405k - 17.2.0 e1d6a67b021e 47deb70fd250 2024-03-21T12:41:14.338 INFO:teuthology.orchestra.run.smithi022.stdout:crash.smithi189 smithi189 running (11m) 43s ago 11m 6405k - 17.2.0 e1d6a67b021e 9a2e0e9b1809 2024-03-21T12:41:14.339 INFO:teuthology.orchestra.run.smithi022.stdout:grafana.smithi022 smithi022 *:3000 error 59s ago 12m - - 2024-03-21T12:41:14.339 INFO:teuthology.orchestra.run.smithi022.stdout:mds.foofs.smithi022.bcnxho smithi022 running (6m) 59s ago 6m 37.7M - 17.2.0 e1d6a67b021e db26b67b8ee7 2024-03-21T12:41:14.339 INFO:teuthology.orchestra.run.smithi022.stdout:mds.foofs.smithi189.lluvko smithi189 running (6m) 43s ago 6m 16.7M - 17.2.0 e1d6a67b021e cf153aff1387 2024-03-21T12:41:14.339 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.smithi022.fektzs smithi022 *:8443,9283,8765 running (2m) 59s ago 13m 478M - 19.0.0-2267-g960fbbe4 19e12293512f d57cebd306b3 2024-03-21T12:41:14.339 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.smithi189.cnxzvu smithi189 *:8443,9283,8765 running (101s) 43s ago 11m 448M - 19.0.0-2267-g960fbbe4 19e12293512f 41663955c56f 2024-03-21T12:41:14.339 INFO:teuthology.orchestra.run.smithi022.stdout:mon.smithi022 smithi022 running (81s) 59s ago 14m 32.2M 2048M 19.0.0-2267-g960fbbe4 19e12293512f c620c9071b2c 2024-03-21T12:41:14.339 INFO:teuthology.orchestra.run.smithi022.stdout:mon.smithi189 smithi189 running (49s) 43s ago 11m 24.5M 2048M 19.0.0-2267-g960fbbe4 19e12293512f b353af3d8f3b 2024-03-21T12:41:14.339 INFO:teuthology.orchestra.run.smithi022.stdout:nfs.foo.0.0.smithi022.ynhbbq smithi022 *:2049 running (6m) 59s ago 6m 258M - 4.0 e1d6a67b021e 33e48e64c60b 2024-03-21T12:41:14.339 INFO:teuthology.orchestra.run.smithi022.stdout:nfs.foo.1.0.smithi189.yzsqml smithi189 *:2049 running (6m) 43s ago 6m 83.3M - 4.0 e1d6a67b021e 48fab2cbdde2 2024-03-21T12:41:14.339 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.smithi022 smithi022 *:9100 running (3m) 59s ago 12m 20.0M - 1.3.1 1dbe0e931976 33287f803f4f 2024-03-21T12:41:14.339 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.smithi189 smithi189 *:9100 running (2m) 43s ago 11m 21.1M - 1.3.1 1dbe0e931976 b51d872e438a 2024-03-21T12:41:14.339 INFO:teuthology.orchestra.run.smithi022.stdout:osd.0 smithi022 running (11m) 59s ago 11m 562M 1937M 17.2.0 e1d6a67b021e 54e6631bd10f 2024-03-21T12:41:14.339 INFO:teuthology.orchestra.run.smithi022.stdout:osd.1 smithi022 running (10m) 59s ago 10m 473M 1937M 17.2.0 e1d6a67b021e 68165c5d09cb 2024-03-21T12:41:14.339 INFO:teuthology.orchestra.run.smithi022.stdout:osd.2 smithi022 running (10m) 59s ago 10m 349M 1937M 17.2.0 e1d6a67b021e fbec9bef8433 2024-03-21T12:41:14.340 INFO:teuthology.orchestra.run.smithi022.stdout:osd.3 smithi022 running (9m) 59s ago 9m 491M 1937M 17.2.0 e1d6a67b021e b1462ffc1b9e 2024-03-21T12:41:14.340 INFO:teuthology.orchestra.run.smithi022.stdout:osd.4 smithi189 running (9m) 43s ago 9m 532M 2704M 17.2.0 e1d6a67b021e 1bedc4eb02ed 2024-03-21T12:41:14.340 INFO:teuthology.orchestra.run.smithi022.stdout:osd.5 smithi189 running (9m) 43s ago 9m 340M 2704M 17.2.0 e1d6a67b021e 25fbab9786a5 2024-03-21T12:41:14.340 INFO:teuthology.orchestra.run.smithi022.stdout:osd.6 smithi189 running (8m) 43s ago 8m 525M 2704M 17.2.0 e1d6a67b021e 0c192fcfd65b 2024-03-21T12:41:14.340 INFO:teuthology.orchestra.run.smithi022.stdout:osd.7 smithi189 running (8m) 43s ago 8m 448M 2704M 17.2.0 e1d6a67b021e 3d0723e1f863 2024-03-21T12:41:14.340 INFO:teuthology.orchestra.run.smithi022.stdout:prometheus.smithi022 smithi022 *:9095 running (108s) 59s ago 11m 65.3M - 2.33.4 514e6a882f6e 8171bbe41d2c 2024-03-21T12:41:14.685 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:14 smithi022 ceph-mon[67047]: pgmap v39: 97 pgs: 97 active+clean; 350 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 9.7 KiB/s rd, 7.7 MiB/s wr, 726 op/s 2024-03-21T12:41:14.685 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:14 smithi022 ceph-mon[67047]: from='client.44125 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:41:14.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:14 smithi189 ceph-mon[53160]: pgmap v39: 97 pgs: 97 active+clean; 350 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 9.7 KiB/s rd, 7.7 MiB/s wr, 726 op/s 2024-03-21T12:41:14.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:14 smithi189 ceph-mon[53160]: from='client.44125 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:41:14.861 INFO:teuthology.orchestra.run.smithi022.stdout: 5 180694 11.33 MB/sec execute 254 sec latency 53.318 ms 2024-03-21T12:41:15.029 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-03-21T12:41:15.029 INFO:teuthology.orchestra.run.smithi022.stdout: "mon": { 2024-03-21T12:41:15.030 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 2 2024-03-21T12:41:15.030 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:41:15.030 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr": { 2024-03-21T12:41:15.030 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 2 2024-03-21T12:41:15.030 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:41:15.030 INFO:teuthology.orchestra.run.smithi022.stdout: "osd": { 2024-03-21T12:41:15.030 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-03-21T12:41:15.030 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:41:15.030 INFO:teuthology.orchestra.run.smithi022.stdout: "mds": { 2024-03-21T12:41:15.030 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-21T12:41:15.030 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:41:15.030 INFO:teuthology.orchestra.run.smithi022.stdout: "overall": { 2024-03-21T12:41:15.030 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-03-21T12:41:15.031 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 4 2024-03-21T12:41:15.031 INFO:teuthology.orchestra.run.smithi022.stdout: } 2024-03-21T12:41:15.031 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-03-21T12:41:15.597 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-03-21T12:41:15.597 INFO:teuthology.orchestra.run.smithi022.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93", 2024-03-21T12:41:15.597 INFO:teuthology.orchestra.run.smithi022.stdout: "in_progress": true, 2024-03-21T12:41:15.598 INFO:teuthology.orchestra.run.smithi022.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-21T12:41:15.598 INFO:teuthology.orchestra.run.smithi022.stdout: "services_complete": [ 2024-03-21T12:41:15.598 INFO:teuthology.orchestra.run.smithi022.stdout: "mon", 2024-03-21T12:41:15.598 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr" 2024-03-21T12:41:15.598 INFO:teuthology.orchestra.run.smithi022.stdout: ], 2024-03-21T12:41:15.598 INFO:teuthology.orchestra.run.smithi022.stdout: "progress": "4/23 daemons upgraded", 2024-03-21T12:41:15.598 INFO:teuthology.orchestra.run.smithi022.stdout: "message": "Currently upgrading crash daemons", 2024-03-21T12:41:15.598 INFO:teuthology.orchestra.run.smithi022.stdout: "is_paused": false 2024-03-21T12:41:15.598 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-03-21T12:41:15.716 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:15 smithi022 ceph-mon[67047]: from='client.34184 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:41:15.717 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:15 smithi022 ceph-mon[67047]: Upgrade: Updating crash.smithi022 (1/2) 2024-03-21T12:41:15.717 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:15 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:41:15.717 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:15 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi022", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-03-21T12:41:15.717 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:15 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:41:15.717 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:15 smithi022 ceph-mon[67047]: Deploying daemon crash.smithi022 on smithi022 2024-03-21T12:41:15.717 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:15 smithi022 ceph-mon[67047]: from='client.? 172.21.15.22:0/2501185133' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:41:15.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:15 smithi189 ceph-mon[53160]: from='client.34184 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:41:15.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:15 smithi189 ceph-mon[53160]: Upgrade: Updating crash.smithi022 (1/2) 2024-03-21T12:41:15.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:15 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:41:15.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:15 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi022", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-03-21T12:41:15.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:15 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:41:15.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:15 smithi189 ceph-mon[53160]: Deploying daemon crash.smithi022 on smithi022 2024-03-21T12:41:15.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:15 smithi189 ceph-mon[53160]: from='client.? 172.21.15.22:0/2501185133' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:41:15.861 INFO:teuthology.orchestra.run.smithi022.stdout: 5 181043 11.33 MB/sec execute 255 sec latency 65.017 ms 2024-03-21T12:41:16.315 INFO:teuthology.orchestra.run.smithi022.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-03-21T12:41:16.316 INFO:teuthology.orchestra.run.smithi022.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-21T12:41:16.316 INFO:teuthology.orchestra.run.smithi022.stdout: daemon grafana.smithi022 on smithi022 is in error state 2024-03-21T12:41:16.729 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:16 smithi022 ceph-mon[67047]: from='client.34192 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:41:16.730 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:16 smithi022 ceph-mon[67047]: pgmap v40: 97 pgs: 97 active+clean; 350 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.4 MiB/s wr, 529 op/s 2024-03-21T12:41:16.730 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:16 smithi022 ceph-mon[67047]: from='client.? 172.21.15.22:0/3675713767' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-21T12:41:16.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:16 smithi189 ceph-mon[53160]: from='client.34192 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:41:16.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:16 smithi189 ceph-mon[53160]: pgmap v40: 97 pgs: 97 active+clean; 350 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.4 MiB/s wr, 529 op/s 2024-03-21T12:41:16.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:16 smithi189 ceph-mon[53160]: from='client.? 172.21.15.22:0/3675713767' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-21T12:41:16.863 INFO:teuthology.orchestra.run.smithi022.stdout: 5 181610 11.36 MB/sec execute 256 sec latency 29.440 ms 2024-03-21T12:41:17.863 INFO:teuthology.orchestra.run.smithi022.stdout: 5 182075 11.38 MB/sec execute 257 sec latency 40.601 ms 2024-03-21T12:41:18.776 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:18 smithi022 ceph-mon[67047]: pgmap v41: 97 pgs: 97 active+clean; 341 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 7.6 MiB/s wr, 765 op/s 2024-03-21T12:41:18.862 INFO:teuthology.orchestra.run.smithi022.stdout: 5 182483 11.39 MB/sec execute 258 sec latency 37.478 ms 2024-03-21T12:41:19.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:18 smithi189 ceph-mon[53160]: pgmap v41: 97 pgs: 97 active+clean; 341 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 7.6 MiB/s wr, 765 op/s 2024-03-21T12:41:19.863 INFO:teuthology.orchestra.run.smithi022.stdout: 5 182876 11.37 MB/sec execute 259 sec latency 33.127 ms 2024-03-21T12:41:20.819 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:20 smithi022 ceph-mon[67047]: pgmap v42: 97 pgs: 97 active+clean; 341 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.0 MiB/s wr, 485 op/s 2024-03-21T12:41:20.862 INFO:teuthology.orchestra.run.smithi022.stdout: 5 183223 11.37 MB/sec execute 260 sec latency 47.881 ms 2024-03-21T12:41:21.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:20 smithi189 ceph-mon[53160]: pgmap v42: 97 pgs: 97 active+clean; 341 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.0 MiB/s wr, 485 op/s 2024-03-21T12:41:21.851 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:21 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:41:21.852 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:21 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:41:21.852 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:21 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:41:21.863 INFO:teuthology.orchestra.run.smithi022.stdout: 5 183497 11.34 MB/sec execute 261 sec latency 63.676 ms 2024-03-21T12:41:22.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:21 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:41:22.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:21 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:41:22.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:21 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:41:22.863 INFO:teuthology.orchestra.run.smithi022.stdout: 5 183829 11.32 MB/sec execute 262 sec latency 54.312 ms 2024-03-21T12:41:23.130 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:23 smithi022 ceph-mon[67047]: pgmap v43: 97 pgs: 97 active+clean; 341 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.0 MiB/s wr, 485 op/s 2024-03-21T12:41:23.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:23 smithi189 ceph-mon[53160]: pgmap v43: 97 pgs: 97 active+clean; 341 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.0 MiB/s wr, 485 op/s 2024-03-21T12:41:23.863 INFO:teuthology.orchestra.run.smithi022.stdout: 5 184136 11.29 MB/sec execute 263 sec latency 52.726 ms 2024-03-21T12:41:24.399 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:24 smithi022 ceph-mon[67047]: Upgrade: Updating crash.smithi189 (2/2) 2024-03-21T12:41:24.400 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:24 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:41:24.400 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:24 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi189", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-03-21T12:41:24.400 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:24 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:41:24.400 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:24 smithi022 ceph-mon[67047]: Deploying daemon crash.smithi189 on smithi189 2024-03-21T12:41:24.441 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:24 smithi189 ceph-mon[53160]: Upgrade: Updating crash.smithi189 (2/2) 2024-03-21T12:41:24.441 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:24 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:41:24.441 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:24 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi189", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-03-21T12:41:24.441 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:24 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:41:24.441 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:24 smithi189 ceph-mon[53160]: Deploying daemon crash.smithi189 on smithi189 2024-03-21T12:41:24.863 INFO:teuthology.orchestra.run.smithi022.stdout: 5 184585 11.31 MB/sec execute 264 sec latency 36.463 ms 2024-03-21T12:41:25.403 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:25 smithi022 ceph-mon[67047]: pgmap v44: 97 pgs: 97 active+clean; 340 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 7.6 MiB/s wr, 730 op/s 2024-03-21T12:41:25.480 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:25 smithi189 ceph-mon[53160]: pgmap v44: 97 pgs: 97 active+clean; 340 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 7.6 MiB/s wr, 730 op/s 2024-03-21T12:41:25.863 INFO:teuthology.orchestra.run.smithi022.stdout: 5 185139 11.34 MB/sec execute 265 sec latency 37.402 ms 2024-03-21T12:41:26.863 INFO:teuthology.orchestra.run.smithi022.stdout: 5 185702 11.36 MB/sec execute 266 sec latency 48.698 ms 2024-03-21T12:41:27.230 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:27 smithi189 ceph-mon[53160]: pgmap v45: 97 pgs: 97 active+clean; 340 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.7 MiB/s wr, 481 op/s 2024-03-21T12:41:27.438 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:27 smithi022 ceph-mon[67047]: pgmap v45: 97 pgs: 97 active+clean; 340 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.7 MiB/s wr, 481 op/s 2024-03-21T12:41:27.863 INFO:teuthology.orchestra.run.smithi022.stdout: 5 186006 11.36 MB/sec execute 267 sec latency 85.963 ms 2024-03-21T12:41:28.475 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:28 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:41:28.475 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:28 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:41:28.475 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:28 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:41:28.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:28 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:41:28.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:28 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:41:28.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:28 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:41:28.863 INFO:teuthology.orchestra.run.smithi022.stdout: 5 186281 11.34 MB/sec execute 268 sec latency 59.682 ms 2024-03-21T12:41:29.491 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:29 smithi189 ceph-mon[53160]: pgmap v46: 97 pgs: 97 active+clean; 346 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 7.3 MiB/s wr, 735 op/s 2024-03-21T12:41:29.496 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:29 smithi022 ceph-mon[67047]: pgmap v46: 97 pgs: 97 active+clean; 346 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 7.3 MiB/s wr, 735 op/s 2024-03-21T12:41:29.863 INFO:teuthology.orchestra.run.smithi022.stdout: 5 186570 11.32 MB/sec execute 269 sec latency 53.631 ms 2024-03-21T12:41:30.863 INFO:teuthology.orchestra.run.smithi022.stdout: 5 186978 11.32 MB/sec execute 270 sec latency 31.626 ms 2024-03-21T12:41:31.264 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:31 smithi022 ceph-mon[67047]: pgmap v47: 97 pgs: 97 active+clean; 346 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.2 MiB/s wr, 499 op/s 2024-03-21T12:41:31.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:31 smithi189 ceph-mon[53160]: pgmap v47: 97 pgs: 97 active+clean; 346 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.2 MiB/s wr, 499 op/s 2024-03-21T12:41:31.863 INFO:teuthology.orchestra.run.smithi022.stdout: 5 187509 11.30 MB/sec execute 271 sec latency 33.945 ms 2024-03-21T12:41:32.783 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:32 smithi022 ceph-mon[67047]: pgmap v48: 97 pgs: 97 active+clean; 346 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.2 MiB/s wr, 499 op/s 2024-03-21T12:41:32.784 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:32 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:41:32.784 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:32 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:41:32.863 INFO:teuthology.orchestra.run.smithi022.stdout: 5 187897 11.29 MB/sec execute 272 sec latency 35.298 ms 2024-03-21T12:41:33.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:32 smithi189 ceph-mon[53160]: pgmap v48: 97 pgs: 97 active+clean; 346 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.2 MiB/s wr, 499 op/s 2024-03-21T12:41:33.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:32 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:41:33.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:32 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:41:33.863 INFO:teuthology.orchestra.run.smithi022.stdout: 5 188396 11.32 MB/sec execute 273 sec latency 40.723 ms 2024-03-21T12:41:34.180 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:34 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:41:34.180 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:34 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:41:34.328 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:34 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:41:34.329 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:34 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:41:34.863 INFO:teuthology.orchestra.run.smithi022.stdout: 5 188833 11.33 MB/sec execute 274 sec latency 55.165 ms 2024-03-21T12:41:35.169 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:35 smithi189 ceph-mon[53160]: pgmap v49: 97 pgs: 97 active+clean; 342 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 10 KiB/s rd, 7.2 MiB/s wr, 721 op/s 2024-03-21T12:41:35.334 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:35 smithi022 ceph-mon[67047]: pgmap v49: 97 pgs: 97 active+clean; 342 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 10 KiB/s rd, 7.2 MiB/s wr, 721 op/s 2024-03-21T12:41:35.863 INFO:teuthology.orchestra.run.smithi022.stdout: 5 189232 11.34 MB/sec execute 275 sec latency 34.223 ms 2024-03-21T12:41:36.345 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:36 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:41:36.345 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:36 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:41:36.346 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:36 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:41:36.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:36 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:41:36.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:36 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:41:36.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:36 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:41:36.863 INFO:teuthology.orchestra.run.smithi022.stdout: 5 189606 11.34 MB/sec execute 276 sec latency 38.926 ms 2024-03-21T12:41:37.361 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:37 smithi022 ceph-mon[67047]: pgmap v50: 97 pgs: 97 active+clean; 342 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.7 MiB/s wr, 476 op/s 2024-03-21T12:41:37.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:37 smithi189 ceph-mon[53160]: pgmap v50: 97 pgs: 97 active+clean; 342 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.7 MiB/s wr, 476 op/s 2024-03-21T12:41:37.863 INFO:teuthology.orchestra.run.smithi022.stdout: 5 189915 11.32 MB/sec execute 277 sec latency 39.213 ms 2024-03-21T12:41:38.378 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:38 smithi022 ceph-mon[67047]: mgrmap e42: smithi022.fektzs(active, since 91s), standbys: smithi189.cnxzvu 2024-03-21T12:41:38.379 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:38 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:41:38.379 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:38 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:41:38.379 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:38 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:41:38.379 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:38 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:41:38.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:38 smithi189 ceph-mon[53160]: mgrmap e42: smithi022.fektzs(active, since 91s), standbys: smithi189.cnxzvu 2024-03-21T12:41:38.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:38 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:41:38.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:38 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:41:38.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:38 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:41:38.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:38 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:41:38.863 INFO:teuthology.orchestra.run.smithi022.stdout: 5 190115 11.30 MB/sec execute 278 sec latency 67.768 ms 2024-03-21T12:41:39.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:39 smithi189 ceph-mon[53160]: pgmap v51: 97 pgs: 97 active+clean; 346 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 9.8 KiB/s rd, 7.5 MiB/s wr, 727 op/s 2024-03-21T12:41:39.410 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:39 smithi022 ceph-mon[67047]: pgmap v51: 97 pgs: 97 active+clean; 346 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 9.8 KiB/s rd, 7.5 MiB/s wr, 727 op/s 2024-03-21T12:41:39.863 INFO:teuthology.orchestra.run.smithi022.stdout: 5 190339 11.28 MB/sec execute 279 sec latency 60.650 ms 2024-03-21T12:41:40.864 INFO:teuthology.orchestra.run.smithi022.stdout: 5 190557 11.26 MB/sec execute 280 sec latency 79.048 ms 2024-03-21T12:41:41.222 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:41 smithi022 ceph-mon[67047]: pgmap v52: 97 pgs: 97 active+clean; 346 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.8 MiB/s wr, 473 op/s 2024-03-21T12:41:41.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:41 smithi189 ceph-mon[53160]: pgmap v52: 97 pgs: 97 active+clean; 346 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.8 MiB/s wr, 473 op/s 2024-03-21T12:41:41.864 INFO:teuthology.orchestra.run.smithi022.stdout: 5 190872 11.23 MB/sec execute 281 sec latency 55.351 ms 2024-03-21T12:41:42.864 INFO:teuthology.orchestra.run.smithi022.stdout: 5 191181 11.21 MB/sec execute 282 sec latency 48.421 ms 2024-03-21T12:41:43.263 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:43 smithi022 ceph-mon[67047]: pgmap v53: 97 pgs: 97 active+clean; 346 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.8 MiB/s wr, 473 op/s 2024-03-21T12:41:43.263 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:43 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:41:43.263 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:43 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:41:43.263 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:43 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:41:43.263 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:43 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:41:43.263 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:43 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:41:43.263 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:43 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:41:43.263 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:43 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:41:43.263 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:43 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:41:43.263 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:43 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:41:43.264 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:43 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:41:43.264 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:43 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:41:43.264 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:43 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi022"}]: dispatch 2024-03-21T12:41:43.264 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:43 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi022"}]': finished 2024-03-21T12:41:43.264 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:43 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi189"}]: dispatch 2024-03-21T12:41:43.264 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:43 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi189"}]': finished 2024-03-21T12:41:43.264 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:43 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-03-21T12:41:43.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:43 smithi189 ceph-mon[53160]: pgmap v53: 97 pgs: 97 active+clean; 346 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.8 MiB/s wr, 473 op/s 2024-03-21T12:41:43.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:43 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:41:43.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:43 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:41:43.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:43 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:41:43.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:43 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:41:43.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:43 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:41:43.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:43 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:41:43.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:43 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:41:43.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:43 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:41:43.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:43 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:41:43.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:43 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:41:43.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:43 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:41:43.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:43 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi022"}]: dispatch 2024-03-21T12:41:43.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:43 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi022"}]': finished 2024-03-21T12:41:43.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:43 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi189"}]: dispatch 2024-03-21T12:41:43.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:43 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi189"}]': finished 2024-03-21T12:41:43.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:43 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-03-21T12:41:43.864 INFO:teuthology.orchestra.run.smithi022.stdout: 5 191426 11.19 MB/sec execute 283 sec latency 52.737 ms 2024-03-21T12:41:44.298 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:44 smithi022 ceph-mon[67047]: pgmap v54: 97 pgs: 97 active+clean; 341 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 6.9 MiB/s wr, 706 op/s 2024-03-21T12:41:44.299 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:44 smithi022 ceph-mon[67047]: pgmap v55: 97 pgs: 97 active+clean; 341 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 9.1 KiB/s rd, 5.7 MiB/s wr, 566 op/s 2024-03-21T12:41:44.299 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:44 smithi022 ceph-mon[67047]: Upgrade: Setting container_image for all crash 2024-03-21T12:41:44.299 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:44 smithi022 ceph-mon[67047]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-03-21T12:41:44.299 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:44 smithi022 ceph-mon[67047]: Upgrade: osd.0 is safe to restart 2024-03-21T12:41:44.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:44 smithi189 ceph-mon[53160]: pgmap v54: 97 pgs: 97 active+clean; 341 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 6.9 MiB/s wr, 706 op/s 2024-03-21T12:41:44.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:44 smithi189 ceph-mon[53160]: pgmap v55: 97 pgs: 97 active+clean; 341 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 9.1 KiB/s rd, 5.7 MiB/s wr, 566 op/s 2024-03-21T12:41:44.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:44 smithi189 ceph-mon[53160]: Upgrade: Setting container_image for all crash 2024-03-21T12:41:44.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:44 smithi189 ceph-mon[53160]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-03-21T12:41:44.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:44 smithi189 ceph-mon[53160]: Upgrade: osd.0 is safe to restart 2024-03-21T12:41:44.864 INFO:teuthology.orchestra.run.smithi022.stdout: 5 191766 11.19 MB/sec execute 284 sec latency 40.494 ms 2024-03-21T12:41:45.864 INFO:teuthology.orchestra.run.smithi022.stdout: 5 192119 11.20 MB/sec execute 285 sec latency 45.292 ms 2024-03-21T12:41:46.335 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:46 smithi022 ceph-mon[67047]: pgmap v56: 97 pgs: 97 active+clean; 341 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 9.1 KiB/s rd, 5.7 MiB/s wr, 566 op/s 2024-03-21T12:41:46.335 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:46 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:41:46.335 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:46 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-03-21T12:41:46.335 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:46 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:41:46.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:46 smithi189 ceph-mon[53160]: pgmap v56: 97 pgs: 97 active+clean; 341 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 9.1 KiB/s rd, 5.7 MiB/s wr, 566 op/s 2024-03-21T12:41:46.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:46 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:41:46.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:46 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-03-21T12:41:46.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:46 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:41:46.864 INFO:teuthology.orchestra.run.smithi022.stdout: 5 192464 11.20 MB/sec execute 286 sec latency 77.698 ms 2024-03-21T12:41:46.937 INFO:teuthology.orchestra.run.smithi022.stdout:true 2024-03-21T12:41:47.353 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:47 smithi022 ceph-mon[67047]: Upgrade: Updating osd.0 2024-03-21T12:41:47.354 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:47 smithi022 ceph-mon[67047]: Deploying daemon osd.0 on smithi022 2024-03-21T12:41:47.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:47 smithi189 ceph-mon[53160]: Upgrade: Updating osd.0 2024-03-21T12:41:47.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:47 smithi189 ceph-mon[53160]: Deploying daemon osd.0 on smithi022 2024-03-21T12:41:47.864 INFO:teuthology.orchestra.run.smithi022.stdout: 5 192793 11.20 MB/sec execute 287 sec latency 79.475 ms 2024-03-21T12:41:48.079 INFO:teuthology.orchestra.run.smithi022.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-21T12:41:48.079 INFO:teuthology.orchestra.run.smithi022.stdout:alertmanager.smithi022 smithi022 *:9093,9094 running (4m) 12s ago 13m 24.1M - 0.23.0 ba2b418f427c 14eb2a3ca318 2024-03-21T12:41:48.079 INFO:teuthology.orchestra.run.smithi022.stdout:crash.smithi022 smithi022 running (27s) 12s ago 13m 6660k - 19.0.0-2267-g960fbbe4 19e12293512f 356b2ddb609a 2024-03-21T12:41:48.079 INFO:teuthology.orchestra.run.smithi022.stdout:crash.smithi189 smithi189 running (21s) 16s ago 12m 6660k - 19.0.0-2267-g960fbbe4 19e12293512f 90ee837f0500 2024-03-21T12:41:48.079 INFO:teuthology.orchestra.run.smithi022.stdout:grafana.smithi022 smithi022 *:3000 error 12s ago 12m - - 2024-03-21T12:41:48.080 INFO:teuthology.orchestra.run.smithi022.stdout:mds.foofs.smithi022.bcnxho smithi022 running (7m) 12s ago 7m 37.9M - 17.2.0 e1d6a67b021e db26b67b8ee7 2024-03-21T12:41:48.080 INFO:teuthology.orchestra.run.smithi022.stdout:mds.foofs.smithi189.lluvko smithi189 running (7m) 16s ago 7m 16.9M - 17.2.0 e1d6a67b021e cf153aff1387 2024-03-21T12:41:48.080 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.smithi022.fektzs smithi022 *:8443,9283,8765 running (3m) 12s ago 14m 500M - 19.0.0-2267-g960fbbe4 19e12293512f d57cebd306b3 2024-03-21T12:41:48.080 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.smithi189.cnxzvu smithi189 *:8443,9283,8765 running (2m) 16s ago 12m 450M - 19.0.0-2267-g960fbbe4 19e12293512f 41663955c56f 2024-03-21T12:41:48.080 INFO:teuthology.orchestra.run.smithi022.stdout:mon.smithi022 smithi022 running (114s) 12s ago 14m 42.7M 2048M 19.0.0-2267-g960fbbe4 19e12293512f c620c9071b2c 2024-03-21T12:41:48.080 INFO:teuthology.orchestra.run.smithi022.stdout:mon.smithi189 smithi189 running (83s) 16s ago 12m 35.8M 2048M 19.0.0-2267-g960fbbe4 19e12293512f b353af3d8f3b 2024-03-21T12:41:48.081 INFO:teuthology.orchestra.run.smithi022.stdout:nfs.foo.0.0.smithi022.ynhbbq smithi022 *:2049 running (7m) 12s ago 6m 258M - 4.0 e1d6a67b021e 33e48e64c60b 2024-03-21T12:41:48.081 INFO:teuthology.orchestra.run.smithi022.stdout:nfs.foo.1.0.smithi189.yzsqml smithi189 *:2049 running (6m) 16s ago 6m 85.4M - 4.0 e1d6a67b021e 48fab2cbdde2 2024-03-21T12:41:48.081 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.smithi022 smithi022 *:9100 running (3m) 12s ago 12m 17.6M - 1.3.1 1dbe0e931976 33287f803f4f 2024-03-21T12:41:48.081 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.smithi189 smithi189 *:9100 running (3m) 16s ago 12m 17.5M - 1.3.1 1dbe0e931976 b51d872e438a 2024-03-21T12:41:48.081 INFO:teuthology.orchestra.run.smithi022.stdout:osd.0 smithi022 running (11m) 12s ago 11m 380M 1937M 17.2.0 e1d6a67b021e 54e6631bd10f 2024-03-21T12:41:48.081 INFO:teuthology.orchestra.run.smithi022.stdout:osd.1 smithi022 running (11m) 12s ago 11m 566M 1937M 17.2.0 e1d6a67b021e 68165c5d09cb 2024-03-21T12:41:48.081 INFO:teuthology.orchestra.run.smithi022.stdout:osd.2 smithi022 running (10m) 12s ago 10m 444M 1937M 17.2.0 e1d6a67b021e fbec9bef8433 2024-03-21T12:41:48.082 INFO:teuthology.orchestra.run.smithi022.stdout:osd.3 smithi022 running (10m) 12s ago 10m 576M 1937M 17.2.0 e1d6a67b021e b1462ffc1b9e 2024-03-21T12:41:48.082 INFO:teuthology.orchestra.run.smithi022.stdout:osd.4 smithi189 running (9m) 16s ago 9m 560M 2704M 17.2.0 e1d6a67b021e 1bedc4eb02ed 2024-03-21T12:41:48.082 INFO:teuthology.orchestra.run.smithi022.stdout:osd.5 smithi189 running (9m) 16s ago 9m 407M 2704M 17.2.0 e1d6a67b021e 25fbab9786a5 2024-03-21T12:41:48.082 INFO:teuthology.orchestra.run.smithi022.stdout:osd.6 smithi189 running (9m) 16s ago 9m 578M 2704M 17.2.0 e1d6a67b021e 0c192fcfd65b 2024-03-21T12:41:48.082 INFO:teuthology.orchestra.run.smithi022.stdout:osd.7 smithi189 running (8m) 16s ago 8m 528M 2704M 17.2.0 e1d6a67b021e 3d0723e1f863 2024-03-21T12:41:48.082 INFO:teuthology.orchestra.run.smithi022.stdout:prometheus.smithi022 smithi022 *:9095 running (2m) 12s ago 12m 73.1M - 2.33.4 514e6a882f6e 8171bbe41d2c 2024-03-21T12:41:48.368 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:48 smithi022 ceph-mon[67047]: pgmap v57: 97 pgs: 97 active+clean; 341 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 4.7 KiB/s rd, 2.0 MiB/s wr, 230 op/s 2024-03-21T12:41:48.368 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:48 smithi022 ceph-mon[67047]: from='client.34200 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:41:48.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:48 smithi189 ceph-mon[53160]: pgmap v57: 97 pgs: 97 active+clean; 341 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 4.7 KiB/s rd, 2.0 MiB/s wr, 230 op/s 2024-03-21T12:41:48.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:48 smithi189 ceph-mon[53160]: from='client.34200 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:41:48.800 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-03-21T12:41:48.800 INFO:teuthology.orchestra.run.smithi022.stdout: "mon": { 2024-03-21T12:41:48.800 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 2 2024-03-21T12:41:48.800 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:41:48.800 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr": { 2024-03-21T12:41:48.800 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 2 2024-03-21T12:41:48.801 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:41:48.801 INFO:teuthology.orchestra.run.smithi022.stdout: "osd": { 2024-03-21T12:41:48.801 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-03-21T12:41:48.801 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:41:48.801 INFO:teuthology.orchestra.run.smithi022.stdout: "mds": { 2024-03-21T12:41:48.801 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-21T12:41:48.801 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:41:48.801 INFO:teuthology.orchestra.run.smithi022.stdout: "overall": { 2024-03-21T12:41:48.801 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-03-21T12:41:48.801 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 4 2024-03-21T12:41:48.801 INFO:teuthology.orchestra.run.smithi022.stdout: } 2024-03-21T12:41:48.801 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-03-21T12:41:48.864 INFO:teuthology.orchestra.run.smithi022.stdout: 5 193125 11.20 MB/sec execute 288 sec latency 33.290 ms 2024-03-21T12:41:49.378 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:49 smithi022 ceph-mon[67047]: from='client.34202 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:41:49.379 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:49 smithi022 ceph-mon[67047]: from='client.34206 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:41:49.379 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:49 smithi022 ceph-mon[67047]: from='client.? 172.21.15.22:0/4058682683' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:41:49.432 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-03-21T12:41:49.433 INFO:teuthology.orchestra.run.smithi022.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93", 2024-03-21T12:41:49.433 INFO:teuthology.orchestra.run.smithi022.stdout: "in_progress": true, 2024-03-21T12:41:49.433 INFO:teuthology.orchestra.run.smithi022.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-21T12:41:49.433 INFO:teuthology.orchestra.run.smithi022.stdout: "services_complete": [ 2024-03-21T12:41:49.433 INFO:teuthology.orchestra.run.smithi022.stdout: "mon", 2024-03-21T12:41:49.433 INFO:teuthology.orchestra.run.smithi022.stdout: "crash", 2024-03-21T12:41:49.433 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr" 2024-03-21T12:41:49.433 INFO:teuthology.orchestra.run.smithi022.stdout: ], 2024-03-21T12:41:49.433 INFO:teuthology.orchestra.run.smithi022.stdout: "progress": "6/23 daemons upgraded", 2024-03-21T12:41:49.433 INFO:teuthology.orchestra.run.smithi022.stdout: "message": "Currently upgrading osd daemons", 2024-03-21T12:41:49.434 INFO:teuthology.orchestra.run.smithi022.stdout: "is_paused": false 2024-03-21T12:41:49.434 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-03-21T12:41:49.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:49 smithi189 ceph-mon[53160]: from='client.34202 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:41:49.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:49 smithi189 ceph-mon[53160]: from='client.34206 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:41:49.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:49 smithi189 ceph-mon[53160]: from='client.? 172.21.15.22:0/4058682683' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:41:49.864 INFO:teuthology.orchestra.run.smithi022.stdout: 5 193407 11.17 MB/sec execute 289 sec latency 53.433 ms 2024-03-21T12:41:49.938 INFO:teuthology.orchestra.run.smithi022.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-03-21T12:41:49.938 INFO:teuthology.orchestra.run.smithi022.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-21T12:41:49.938 INFO:teuthology.orchestra.run.smithi022.stdout: daemon grafana.smithi022 on smithi022 is in error state 2024-03-21T12:41:50.405 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:50 smithi022 ceph-mon[67047]: pgmap v58: 97 pgs: 97 active+clean; 341 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 7.9 KiB/s rd, 4.5 MiB/s wr, 476 op/s 2024-03-21T12:41:50.406 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:50 smithi022 ceph-mon[67047]: from='client.? 172.21.15.22:0/1624824958' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-21T12:41:50.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:50 smithi189 ceph-mon[53160]: pgmap v58: 97 pgs: 97 active+clean; 341 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 7.9 KiB/s rd, 4.5 MiB/s wr, 476 op/s 2024-03-21T12:41:50.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:50 smithi189 ceph-mon[53160]: from='client.? 172.21.15.22:0/1624824958' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-21T12:41:50.864 INFO:teuthology.orchestra.run.smithi022.stdout: 5 193652 11.16 MB/sec execute 290 sec latency 61.794 ms 2024-03-21T12:41:50.922 INFO:journalctl@ceph.osd.0.smithi022.stdout:Mar 21 12:41:50 smithi022 systemd[1]: Stopping Ceph osd.0 for 3fc7f0f2-e77e-11ee-95cd-87774f69a715... 2024-03-21T12:41:51.174 INFO:journalctl@ceph.osd.0.smithi022.stdout:Mar 21 12:41:51 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-0[34895]: 2024-03-21T12:41:51.108+0000 7f78f2c21700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-21T12:41:51.174 INFO:journalctl@ceph.osd.0.smithi022.stdout:Mar 21 12:41:51 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-0[34895]: 2024-03-21T12:41:51.108+0000 7f78f2c21700 -1 osd.0 73 *** Got signal Terminated *** 2024-03-21T12:41:51.174 INFO:journalctl@ceph.osd.0.smithi022.stdout:Mar 21 12:41:51 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-0[34895]: 2024-03-21T12:41:51.108+0000 7f78f2c21700 -1 osd.0 73 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-21T12:41:51.427 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:51 smithi022 ceph-mon[67047]: from='client.34214 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:41:51.428 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:51 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:41:51.428 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:51 smithi022 ceph-mon[67047]: osd.0 marked itself down and dead 2024-03-21T12:41:51.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:51 smithi189 ceph-mon[53160]: from='client.34214 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:41:51.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:51 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:41:51.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:51 smithi189 ceph-mon[53160]: osd.0 marked itself down and dead 2024-03-21T12:41:51.864 INFO:teuthology.orchestra.run.smithi022.stdout: 5 193855 11.13 MB/sec execute 291 sec latency 627.544 ms 2024-03-21T12:41:52.094 INFO:journalctl@ceph.osd.0.smithi022.stdout:Mar 21 12:41:51 smithi022 podman[72632]: 2024-03-21 12:41:51.880954231 +0000 UTC m=+0.926947783 container died 54e6631bd10f685ee5bf8b852453084918375a48cf700c6221b35481f012401e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-0, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, release=754, architecture=x86_64, vcs-type=git, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, GIT_CLEAN=True, distribution-scope=public, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, GIT_BRANCH=HEAD, ceph=True) 2024-03-21T12:41:52.428 INFO:journalctl@ceph.osd.0.smithi022.stdout:Mar 21 12:41:52 smithi022 podman[72632]: 2024-03-21 12:41:52.294015709 +0000 UTC m=+1.340009265 container cleanup 54e6631bd10f685ee5bf8b852453084918375a48cf700c6221b35481f012401e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-0, name=centos-stream, release=754, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, ceph=True, architecture=x86_64, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, version=8, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0) 2024-03-21T12:41:52.428 INFO:journalctl@ceph.osd.0.smithi022.stdout:Mar 21 12:41:52 smithi022 bash[72632]: ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-0 2024-03-21T12:41:52.429 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:52 smithi022 ceph-mon[67047]: pgmap v59: 97 pgs: 97 active+clean; 341 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 7.9 KiB/s rd, 4.5 MiB/s wr, 476 op/s 2024-03-21T12:41:52.429 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:52 smithi022 ceph-mon[67047]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-21T12:41:52.429 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:52 smithi022 ceph-mon[67047]: osdmap e74: 8 total, 7 up, 8 in 2024-03-21T12:41:52.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:52 smithi189 ceph-mon[53160]: pgmap v59: 97 pgs: 97 active+clean; 341 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 7.9 KiB/s rd, 4.5 MiB/s wr, 476 op/s 2024-03-21T12:41:52.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:52 smithi189 ceph-mon[53160]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-21T12:41:52.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:52 smithi189 ceph-mon[53160]: osdmap e74: 8 total, 7 up, 8 in 2024-03-21T12:41:52.684 INFO:journalctl@ceph.osd.0.smithi022.stdout:Mar 21 12:41:52 smithi022 podman[72650]: 2024-03-21 12:41:52.616615194 +0000 UTC m=+0.734583900 container remove 54e6631bd10f685ee5bf8b852453084918375a48cf700c6221b35481f012401e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, vcs-type=git, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, com.redhat.component=centos-stream-container, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, name=centos-stream, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-03-21T12:41:52.864 INFO:teuthology.orchestra.run.smithi022.stdout: 5 194060 11.13 MB/sec execute 292 sec latency 1017.065 ms 2024-03-21T12:41:53.187 INFO:journalctl@ceph.osd.0.smithi022.stdout:Mar 21 12:41:53 smithi022 podman[72734]: 2024-03-21 12:41:53.088781086 +0000 UTC m=+0.030142123 image pull 19e12293512ff7978c8adb5f72a829060dd47f07bfd6903b6b3178d2ebc29e4e quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93 2024-03-21T12:41:53.439 INFO:journalctl@ceph.osd.0.smithi022.stdout:Mar 21 12:41:53 smithi022 podman[72734]: 2024-03-21 12:41:53.214944751 +0000 UTC m=+0.156305763 container create 0ab0abbe162c13f46ac5e286add25dc475cfc838c74f3c898dd73ec780628515 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-0-deactivate, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, ceph=True, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240319) 2024-03-21T12:41:53.439 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:53 smithi022 ceph-mon[67047]: osdmap e75: 8 total, 7 up, 8 in 2024-03-21T12:41:53.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:53 smithi189 ceph-mon[53160]: osdmap e75: 8 total, 7 up, 8 in 2024-03-21T12:41:53.690 INFO:journalctl@ceph.osd.0.smithi022.stdout:Mar 21 12:41:53 smithi022 podman[72734]: 2024-03-21 12:41:53.617807078 +0000 UTC m=+0.559168106 container init 0ab0abbe162c13f46ac5e286add25dc475cfc838c74f3c898dd73ec780628515 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-0-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, GIT_CLEAN=True, org.label-schema.build-date=20240319, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, ceph=True) 2024-03-21T12:41:53.690 INFO:journalctl@ceph.osd.0.smithi022.stdout:Mar 21 12:41:53 smithi022 podman[72734]: 2024-03-21 12:41:53.623365539 +0000 UTC m=+0.564726556 container start 0ab0abbe162c13f46ac5e286add25dc475cfc838c74f3c898dd73ec780628515 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-0-deactivate, org.label-schema.build-date=20240319, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, ceph=True, org.label-schema.license=GPLv2, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-03-21T12:41:53.864 INFO:teuthology.orchestra.run.smithi022.stdout: 5 194414 11.10 MB/sec execute 293 sec latency 60.867 ms 2024-03-21T12:41:53.943 INFO:journalctl@ceph.osd.0.smithi022.stdout:Mar 21 12:41:53 smithi022 podman[72734]: 2024-03-21 12:41:53.743062573 +0000 UTC m=+0.684423587 container attach 0ab0abbe162c13f46ac5e286add25dc475cfc838c74f3c898dd73ec780628515 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-0-deactivate, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240319, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-03-21T12:41:54.202 INFO:journalctl@ceph.osd.0.smithi022.stdout:Mar 21 12:41:54 smithi022 podman[72734]: 2024-03-21 12:41:54.078261341 +0000 UTC m=+1.019622356 container died 0ab0abbe162c13f46ac5e286add25dc475cfc838c74f3c898dd73ec780628515 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-0-deactivate, org.label-schema.vendor=CentOS, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240319, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-03-21T12:41:54.452 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:54 smithi022 ceph-mon[67047]: pgmap v62: 97 pgs: 26 peering, 71 active+clean; 350 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 6.6 KiB/s rd, 6.0 MiB/s wr, 555 op/s 2024-03-21T12:41:54.453 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:54 smithi022 ceph-mon[67047]: Health check failed: Reduced data availability: 1 pg inactive, 7 pgs peering (PG_AVAILABILITY) 2024-03-21T12:41:54.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:54 smithi189 ceph-mon[53160]: pgmap v62: 97 pgs: 26 peering, 71 active+clean; 350 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 6.6 KiB/s rd, 6.0 MiB/s wr, 555 op/s 2024-03-21T12:41:54.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:54 smithi189 ceph-mon[53160]: Health check failed: Reduced data availability: 1 pg inactive, 7 pgs peering (PG_AVAILABILITY) 2024-03-21T12:41:54.864 INFO:teuthology.orchestra.run.smithi022.stdout: 5 194761 11.08 MB/sec execute 294 sec latency 39.960 ms 2024-03-21T12:41:55.214 INFO:journalctl@ceph.osd.0.smithi022.stdout:Mar 21 12:41:55 smithi022 podman[72761]: 2024-03-21 12:41:55.126050457 +0000 UTC m=+1.038265358 container remove 0ab0abbe162c13f46ac5e286add25dc475cfc838c74f3c898dd73ec780628515 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-0-deactivate, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, GIT_BRANCH=HEAD, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240319, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True) 2024-03-21T12:41:55.214 INFO:journalctl@ceph.osd.0.smithi022.stdout:Mar 21 12:41:55 smithi022 systemd[1]: ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715@osd.0.service: Deactivated successfully. 2024-03-21T12:41:55.214 INFO:journalctl@ceph.osd.0.smithi022.stdout:Mar 21 12:41:55 smithi022 systemd[1]: Stopped Ceph osd.0 for 3fc7f0f2-e77e-11ee-95cd-87774f69a715. 2024-03-21T12:41:55.214 INFO:journalctl@ceph.osd.0.smithi022.stdout:Mar 21 12:41:55 smithi022 systemd[1]: ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715@osd.0.service: Consumed 3min 26.777s CPU time. 2024-03-21T12:41:55.719 INFO:journalctl@ceph.osd.0.smithi022.stdout:Mar 21 12:41:55 smithi022 systemd[1]: Starting Ceph osd.0 for 3fc7f0f2-e77e-11ee-95cd-87774f69a715... 2024-03-21T12:41:55.871 INFO:teuthology.orchestra.run.smithi022.stdout: 5 195224 11.09 MB/sec execute 295 sec latency 33.417 ms 2024-03-21T12:41:55.970 INFO:journalctl@ceph.osd.0.smithi022.stdout:Mar 21 12:41:55 smithi022 podman[72860]: 2024-03-21 12:41:55.640305801 +0000 UTC m=+0.034513940 image pull 19e12293512ff7978c8adb5f72a829060dd47f07bfd6903b6b3178d2ebc29e4e quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93 2024-03-21T12:41:55.970 INFO:journalctl@ceph.osd.0.smithi022.stdout:Mar 21 12:41:55 smithi022 podman[72860]: 2024-03-21 12:41:55.797214736 +0000 UTC m=+0.191422921 container create 810f92ca042a8dbaff828b4af11830ad6c8f107611dc8a636e09abd3179b7453 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-0-activate, org.label-schema.build-date=20240319, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe) 2024-03-21T12:41:56.222 INFO:journalctl@ceph.osd.0.smithi022.stdout:Mar 21 12:41:56 smithi022 podman[72860]: 2024-03-21 12:41:56.195568305 +0000 UTC m=+0.589776447 container init 810f92ca042a8dbaff828b4af11830ad6c8f107611dc8a636e09abd3179b7453 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-0-activate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.build-date=20240319, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, GIT_CLEAN=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-03-21T12:41:56.223 INFO:journalctl@ceph.osd.0.smithi022.stdout:Mar 21 12:41:56 smithi022 podman[72860]: 2024-03-21 12:41:56.204490678 +0000 UTC m=+0.598698810 container start 810f92ca042a8dbaff828b4af11830ad6c8f107611dc8a636e09abd3179b7453 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-0-activate, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.build-date=20240319, ceph=True, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-03-21T12:41:56.479 INFO:journalctl@ceph.osd.0.smithi022.stdout:Mar 21 12:41:56 smithi022 podman[72860]: 2024-03-21 12:41:56.37271126 +0000 UTC m=+0.766919391 container attach 810f92ca042a8dbaff828b4af11830ad6c8f107611dc8a636e09abd3179b7453 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-0-activate, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.build-date=20240319, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-03-21T12:41:56.479 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:56 smithi022 ceph-mon[67047]: pgmap v63: 97 pgs: 26 peering, 71 active+clean; 350 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 6.6 KiB/s rd, 6.0 MiB/s wr, 555 op/s 2024-03-21T12:41:56.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:56 smithi189 ceph-mon[53160]: pgmap v63: 97 pgs: 26 peering, 71 active+clean; 350 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 6.6 KiB/s rd, 6.0 MiB/s wr, 555 op/s 2024-03-21T12:41:56.871 INFO:teuthology.orchestra.run.smithi022.stdout: 5 195762 11.11 MB/sec execute 296 sec latency 47.803 ms 2024-03-21T12:41:56.981 INFO:journalctl@ceph.osd.0.smithi022.stdout:Mar 21 12:41:56 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-0-activate[72873]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-03-21T12:41:56.981 INFO:journalctl@ceph.osd.0.smithi022.stdout:Mar 21 12:41:56 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-0-activate[72873]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-03-21T12:41:56.981 INFO:journalctl@ceph.osd.0.smithi022.stdout:Mar 21 12:41:56 smithi022 bash[72860]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-03-21T12:41:56.981 INFO:journalctl@ceph.osd.0.smithi022.stdout:Mar 21 12:41:56 smithi022 bash[72860]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-03-21T12:41:56.981 INFO:journalctl@ceph.osd.0.smithi022.stdout:Mar 21 12:41:56 smithi022 bash[72860]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-03-21T12:41:56.981 INFO:journalctl@ceph.osd.0.smithi022.stdout:Mar 21 12:41:56 smithi022 bash[72860]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-03-21T12:41:56.981 INFO:journalctl@ceph.osd.0.smithi022.stdout:Mar 21 12:41:56 smithi022 bash[72860]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-03-21T12:41:56.982 INFO:journalctl@ceph.osd.0.smithi022.stdout:Mar 21 12:41:56 smithi022 bash[72860]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-03-21T12:41:56.982 INFO:journalctl@ceph.osd.0.smithi022.stdout:Mar 21 12:41:56 smithi022 bash[72860]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-03-21T12:41:56.982 INFO:journalctl@ceph.osd.0.smithi022.stdout:Mar 21 12:41:56 smithi022 bash[72860]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-03-21T12:41:56.982 INFO:journalctl@ceph.osd.0.smithi022.stdout:Mar 21 12:41:56 smithi022 bash[72860]: --> ceph-volume raw activate successful for osd ID: 0 2024-03-21T12:41:56.982 INFO:journalctl@ceph.osd.0.smithi022.stdout:Mar 21 12:41:56 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-0-activate[72873]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-03-21T12:41:56.982 INFO:journalctl@ceph.osd.0.smithi022.stdout:Mar 21 12:41:56 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-0-activate[72873]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-03-21T12:41:56.982 INFO:journalctl@ceph.osd.0.smithi022.stdout:Mar 21 12:41:56 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-0-activate[72873]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-03-21T12:41:56.982 INFO:journalctl@ceph.osd.0.smithi022.stdout:Mar 21 12:41:56 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-0-activate[72873]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-03-21T12:41:56.982 INFO:journalctl@ceph.osd.0.smithi022.stdout:Mar 21 12:41:56 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-0-activate[72873]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-03-21T12:41:56.982 INFO:journalctl@ceph.osd.0.smithi022.stdout:Mar 21 12:41:56 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-0-activate[72873]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-03-21T12:41:56.982 INFO:journalctl@ceph.osd.0.smithi022.stdout:Mar 21 12:41:56 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-0-activate[72873]: --> ceph-volume raw activate successful for osd ID: 0 2024-03-21T12:41:57.036 INFO:journalctl@ceph.osd.0.smithi022.stdout:Mar 21 12:41:56 smithi022 podman[73004]: 2024-03-21 12:41:56.982147491 +0000 UTC m=+0.027627652 container died 810f92ca042a8dbaff828b4af11830ad6c8f107611dc8a636e09abd3179b7453 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-0-activate, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.build-date=20240319, CEPH_POINT_RELEASE=, ceph=True, GIT_BRANCH=HEAD, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-03-21T12:41:57.872 INFO:teuthology.orchestra.run.smithi022.stdout: 5 196349 11.15 MB/sec execute 297 sec latency 32.957 ms 2024-03-21T12:41:57.984 INFO:journalctl@ceph.osd.0.smithi022.stdout:Mar 21 12:41:57 smithi022 podman[73004]: 2024-03-21 12:41:57.96904147 +0000 UTC m=+1.014521632 container remove 810f92ca042a8dbaff828b4af11830ad6c8f107611dc8a636e09abd3179b7453 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-0-activate, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, GIT_CLEAN=True, org.label-schema.build-date=20240319, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.33.5) 2024-03-21T12:41:58.236 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:58 smithi022 ceph-mon[67047]: pgmap v64: 97 pgs: 26 peering, 71 active+clean; 350 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 3.0 KiB/s rd, 3.2 MiB/s wr, 280 op/s 2024-03-21T12:41:58.486 INFO:journalctl@ceph.osd.0.smithi022.stdout:Mar 21 12:41:58 smithi022 podman[73058]: 2024-03-21 12:41:58.159141893 +0000 UTC m=+0.032385073 image pull 19e12293512ff7978c8adb5f72a829060dd47f07bfd6903b6b3178d2ebc29e4e quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93 2024-03-21T12:41:58.487 INFO:journalctl@ceph.osd.0.smithi022.stdout:Mar 21 12:41:58 smithi022 podman[73058]: 2024-03-21 12:41:58.353719121 +0000 UTC m=+0.226962284 container create 23a85645eeea64275ee72959e9c651764187c829fa448bc8a3633f97238da164 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-0, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.build-date=20240319, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-03-21T12:41:58.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:58 smithi189 ceph-mon[53160]: pgmap v64: 97 pgs: 26 peering, 71 active+clean; 350 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 3.0 KiB/s rd, 3.2 MiB/s wr, 280 op/s 2024-03-21T12:41:58.873 INFO:teuthology.orchestra.run.smithi022.stdout: 5 196820 11.15 MB/sec execute 298 sec latency 23.939 ms 2024-03-21T12:41:58.989 INFO:journalctl@ceph.osd.0.smithi022.stdout:Mar 21 12:41:58 smithi022 podman[73058]: 2024-03-21 12:41:58.749368729 +0000 UTC m=+0.622611897 container init 23a85645eeea64275ee72959e9c651764187c829fa448bc8a3633f97238da164 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240319, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe) 2024-03-21T12:41:58.989 INFO:journalctl@ceph.osd.0.smithi022.stdout:Mar 21 12:41:58 smithi022 podman[73058]: 2024-03-21 12:41:58.756287907 +0000 UTC m=+0.629531070 container start 23a85645eeea64275ee72959e9c651764187c829fa448bc8a3633f97238da164 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, org.label-schema.build-date=20240319, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, ceph=True) 2024-03-21T12:41:58.989 INFO:journalctl@ceph.osd.0.smithi022.stdout:Mar 21 12:41:58 smithi022 ceph-osd[73110]: -- 172.21.15.22:0/4096842539 <== mon.1 v2:172.21.15.189:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55611b0e5680 con 0x55611b0af000 2024-03-21T12:41:58.989 INFO:journalctl@ceph.osd.0.smithi022.stdout:Mar 21 12:41:58 smithi022 bash[73058]: 23a85645eeea64275ee72959e9c651764187c829fa448bc8a3633f97238da164 2024-03-21T12:41:58.989 INFO:journalctl@ceph.osd.0.smithi022.stdout:Mar 21 12:41:58 smithi022 systemd[1]: Started Ceph osd.0 for 3fc7f0f2-e77e-11ee-95cd-87774f69a715. 2024-03-21T12:41:59.495 INFO:journalctl@ceph.osd.0.smithi022.stdout:Mar 21 12:41:59 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-0[73106]: 2024-03-21T12:41:59.337+0000 7f7cbf2ff7c0 -1 Falling back to public interface 2024-03-21T12:41:59.873 INFO:teuthology.orchestra.run.smithi022.stdout: 5 197209 11.14 MB/sec execute 299 sec latency 29.689 ms 2024-03-21T12:41:59.999 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:59 smithi022 ceph-mon[67047]: pgmap v65: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 345 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 9.0 KiB/s rd, 6.2 MiB/s wr, 611 op/s; 216/1641 objects degraded (13.163%) 2024-03-21T12:41:59.999 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:59 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:41:59.999 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:59 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:41:59.999 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:59 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:41:59.999 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:59 smithi022 ceph-mon[67047]: Health check failed: Degraded data redundancy: 216/1641 objects degraded (13.163%), 29 pgs degraded (PG_DEGRADED) 2024-03-21T12:41:59.999 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:41:59 smithi022 ceph-mon[67047]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 7 pgs peering) 2024-03-21T12:42:00.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:59 smithi189 ceph-mon[53160]: pgmap v65: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 345 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 9.0 KiB/s rd, 6.2 MiB/s wr, 611 op/s; 216/1641 objects degraded (13.163%) 2024-03-21T12:42:00.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:59 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:42:00.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:59 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:42:00.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:59 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:42:00.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:59 smithi189 ceph-mon[53160]: Health check failed: Degraded data redundancy: 216/1641 objects degraded (13.163%), 29 pgs degraded (PG_DEGRADED) 2024-03-21T12:42:00.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:41:59 smithi189 ceph-mon[53160]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 7 pgs peering) 2024-03-21T12:42:00.873 INFO:teuthology.orchestra.run.smithi022.stdout: 5 197724 11.15 MB/sec execute 300 sec latency 58.892 ms 2024-03-21T12:42:01.873 INFO:teuthology.orchestra.run.smithi022.stdout: 5 198174 11.13 MB/sec execute 301 sec latency 61.071 ms 2024-03-21T12:42:02.040 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:01 smithi022 ceph-mon[67047]: pgmap v66: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 345 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.3 MiB/s wr, 520 op/s; 216/1641 objects degraded (13.163%) 2024-03-21T12:42:02.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:01 smithi189 ceph-mon[53160]: pgmap v66: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 345 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.3 MiB/s wr, 520 op/s; 216/1641 objects degraded (13.163%) 2024-03-21T12:42:02.874 INFO:teuthology.orchestra.run.smithi022.stdout: 5 198486 11.12 MB/sec execute 302 sec latency 64.369 ms 2024-03-21T12:42:03.874 INFO:teuthology.orchestra.run.smithi022.stdout: 5 198947 11.14 MB/sec execute 303 sec latency 32.126 ms 2024-03-21T12:42:04.055 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:04 smithi022 ceph-mon[67047]: pgmap v67: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 344 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 9.4 KiB/s rd, 5.6 MiB/s wr, 595 op/s; 227/1680 objects degraded (13.512%) 2024-03-21T12:42:04.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:04 smithi189 ceph-mon[53160]: pgmap v67: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 344 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 9.4 KiB/s rd, 5.6 MiB/s wr, 595 op/s; 227/1680 objects degraded (13.512%) 2024-03-21T12:42:04.874 INFO:teuthology.orchestra.run.smithi022.stdout: 5 199285 11.14 MB/sec execute 304 sec latency 45.092 ms 2024-03-21T12:42:05.100 INFO:journalctl@ceph.osd.0.smithi022.stdout:Mar 21 12:42:05 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-0[73106]: 2024-03-21T12:42:05.080+0000 7f7cbf2ff7c0 -1 osd.0 0 read_superblock omap replica is missing. 2024-03-21T12:42:05.874 INFO:teuthology.orchestra.run.smithi022.stdout: 5 199681 11.15 MB/sec execute 305 sec latency 57.790 ms 2024-03-21T12:42:06.116 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:06 smithi022 ceph-mon[67047]: pgmap v68: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 344 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.9 MiB/s wr, 516 op/s; 227/1680 objects degraded (13.512%) 2024-03-21T12:42:06.117 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:06 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:42:06.117 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:06 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:42:06.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:06 smithi189 ceph-mon[53160]: pgmap v68: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 344 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.9 MiB/s wr, 516 op/s; 227/1680 objects degraded (13.512%) 2024-03-21T12:42:06.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:06 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:42:06.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:06 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:42:06.633 INFO:journalctl@ceph.osd.0.smithi022.stdout:Mar 21 12:42:06 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-0[73106]: 2024-03-21T12:42:06.516+0000 7f7cbf2ff7c0 -1 osd.0 73 log_to_monitors true 2024-03-21T12:42:06.874 INFO:teuthology.orchestra.run.smithi022.stdout: 5 199940 11.13 MB/sec execute 306 sec latency 53.372 ms 2024-03-21T12:42:07.139 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:07 smithi022 ceph-mon[67047]: from='osd.0 [v2:172.21.15.22:6802/3188193291,v1:172.21.15.22:6803/3188193291]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-03-21T12:42:07.140 INFO:journalctl@ceph.osd.0.smithi022.stdout:Mar 21 12:42:07 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-0[73106]: 2024-03-21T12:42:07.050+0000 7f7cb0757700 -1 osd.0 73 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-21T12:42:07.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:07 smithi189 ceph-mon[53160]: from='osd.0 [v2:172.21.15.22:6802/3188193291,v1:172.21.15.22:6803/3188193291]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-03-21T12:42:07.874 INFO:teuthology.orchestra.run.smithi022.stdout: 5 200175 11.12 MB/sec execute 307 sec latency 45.481 ms 2024-03-21T12:42:08.176 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:08 smithi022 ceph-mon[67047]: pgmap v69: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 344 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.9 MiB/s wr, 516 op/s; 227/1680 objects degraded (13.512%) 2024-03-21T12:42:08.177 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:08 smithi022 ceph-mon[67047]: from='osd.0 [v2:172.21.15.22:6802/3188193291,v1:172.21.15.22:6803/3188193291]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-03-21T12:42:08.177 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:08 smithi022 ceph-mon[67047]: osdmap e76: 8 total, 7 up, 8 in 2024-03-21T12:42:08.177 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:08 smithi022 ceph-mon[67047]: from='osd.0 [v2:172.21.15.22:6802/3188193291,v1:172.21.15.22:6803/3188193291]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]: dispatch 2024-03-21T12:42:08.177 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:08 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:42:08.177 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:08 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:42:08.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:08 smithi189 ceph-mon[53160]: pgmap v69: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 344 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.9 MiB/s wr, 516 op/s; 227/1680 objects degraded (13.512%) 2024-03-21T12:42:08.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:08 smithi189 ceph-mon[53160]: from='osd.0 [v2:172.21.15.22:6802/3188193291,v1:172.21.15.22:6803/3188193291]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-03-21T12:42:08.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:08 smithi189 ceph-mon[53160]: osdmap e76: 8 total, 7 up, 8 in 2024-03-21T12:42:08.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:08 smithi189 ceph-mon[53160]: from='osd.0 [v2:172.21.15.22:6802/3188193291,v1:172.21.15.22:6803/3188193291]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]: dispatch 2024-03-21T12:42:08.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:08 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:42:08.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:08 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:42:08.874 INFO:teuthology.orchestra.run.smithi022.stdout: 5 200314 11.08 MB/sec execute 308 sec latency 849.138 ms 2024-03-21T12:42:09.346 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:09 smithi022 ceph-mon[67047]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-21T12:42:09.346 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:09 smithi022 ceph-mon[67047]: osd.0 [v2:172.21.15.22:6802/3188193291,v1:172.21.15.22:6803/3188193291] boot 2024-03-21T12:42:09.346 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:09 smithi022 ceph-mon[67047]: osdmap e77: 8 total, 8 up, 8 in 2024-03-21T12:42:09.346 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:09 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-21T12:42:09.346 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:09 smithi022 ceph-mon[67047]: Health check update: Degraded data redundancy: 227/1680 objects degraded (13.512%), 29 pgs degraded (PG_DEGRADED) 2024-03-21T12:42:09.346 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:09 smithi022 ceph-mon[67047]: osdmap e78: 8 total, 8 up, 8 in 2024-03-21T12:42:09.346 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:09 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:42:09.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:09 smithi189 ceph-mon[53160]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-21T12:42:09.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:09 smithi189 ceph-mon[53160]: osd.0 [v2:172.21.15.22:6802/3188193291,v1:172.21.15.22:6803/3188193291] boot 2024-03-21T12:42:09.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:09 smithi189 ceph-mon[53160]: osdmap e77: 8 total, 8 up, 8 in 2024-03-21T12:42:09.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:09 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-21T12:42:09.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:09 smithi189 ceph-mon[53160]: Health check update: Degraded data redundancy: 227/1680 objects degraded (13.512%), 29 pgs degraded (PG_DEGRADED) 2024-03-21T12:42:09.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:09 smithi189 ceph-mon[53160]: osdmap e78: 8 total, 8 up, 8 in 2024-03-21T12:42:09.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:09 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:42:09.874 INFO:teuthology.orchestra.run.smithi022.stdout: 5 200322 11.05 MB/sec execute 309 sec latency 1849.184 ms 2024-03-21T12:42:10.271 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:10 smithi022 ceph-mon[67047]: pgmap v72: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 345 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.9 MiB/s wr, 768 op/s; 212/1653 objects degraded (12.825%) 2024-03-21T12:42:10.272 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:10 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:42:10.272 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:10 smithi022 ceph-mon[67047]: osdmap e79: 8 total, 8 up, 8 in 2024-03-21T12:42:10.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:10 smithi189 ceph-mon[53160]: pgmap v72: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 345 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.9 MiB/s wr, 768 op/s; 212/1653 objects degraded (12.825%) 2024-03-21T12:42:10.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:10 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:42:10.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:10 smithi189 ceph-mon[53160]: osdmap e79: 8 total, 8 up, 8 in 2024-03-21T12:42:10.874 INFO:teuthology.orchestra.run.smithi022.stdout: 5 200471 11.03 MB/sec execute 310 sec latency 2215.842 ms 2024-03-21T12:42:11.874 INFO:teuthology.orchestra.run.smithi022.stdout: 5 200687 11.01 MB/sec execute 311 sec latency 42.092 ms 2024-03-21T12:42:12.082 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:12 smithi022 ceph-mon[67047]: pgmap v75: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 345 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 7.4 MiB/s wr, 649 op/s; 212/1653 objects degraded (12.825%) 2024-03-21T12:42:12.082 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:12 smithi022 ceph-mon[67047]: osdmap e80: 8 total, 8 up, 8 in 2024-03-21T12:42:12.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:12 smithi189 ceph-mon[53160]: pgmap v75: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 345 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 7.4 MiB/s wr, 649 op/s; 212/1653 objects degraded (12.825%) 2024-03-21T12:42:12.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:12 smithi189 ceph-mon[53160]: osdmap e80: 8 total, 8 up, 8 in 2024-03-21T12:42:12.874 INFO:teuthology.orchestra.run.smithi022.stdout: 5 200964 11.00 MB/sec execute 312 sec latency 47.225 ms 2024-03-21T12:42:13.114 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:13 smithi022 ceph-mon[67047]: osdmap e81: 8 total, 8 up, 8 in 2024-03-21T12:42:13.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:13 smithi189 ceph-mon[53160]: osdmap e81: 8 total, 8 up, 8 in 2024-03-21T12:42:13.874 INFO:teuthology.orchestra.run.smithi022.stdout: 5 201316 11.00 MB/sec execute 313 sec latency 807.556 ms 2024-03-21T12:42:14.134 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:14 smithi022 ceph-mon[67047]: pgmap v78: 97 pgs: 87 active+clean, 3 peering, 1 active+recovery_wait+degraded, 1 active+recovering+degraded, 5 active+recovery_wait+undersized+degraded+remapped; 337 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 3.0 MiB/s wr, 334 op/s; 157/1470 objects degraded (10.680%); 2.0 MiB/s, 31 objects/s recovering 2024-03-21T12:42:14.134 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:14 smithi022 ceph-mon[67047]: osdmap e82: 8 total, 8 up, 8 in 2024-03-21T12:42:14.134 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:14 smithi022 ceph-mon[67047]: Health check update: Degraded data redundancy: 157/1470 objects degraded (10.680%), 7 pgs degraded (PG_DEGRADED) 2024-03-21T12:42:14.135 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:14 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:42:14.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:14 smithi189 ceph-mon[53160]: pgmap v78: 97 pgs: 87 active+clean, 3 peering, 1 active+recovery_wait+degraded, 1 active+recovering+degraded, 5 active+recovery_wait+undersized+degraded+remapped; 337 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 3.0 MiB/s wr, 334 op/s; 157/1470 objects degraded (10.680%); 2.0 MiB/s, 31 objects/s recovering 2024-03-21T12:42:14.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:14 smithi189 ceph-mon[53160]: osdmap e82: 8 total, 8 up, 8 in 2024-03-21T12:42:14.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:14 smithi189 ceph-mon[53160]: Health check update: Degraded data redundancy: 157/1470 objects degraded (10.680%), 7 pgs degraded (PG_DEGRADED) 2024-03-21T12:42:14.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:14 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:42:14.874 INFO:teuthology.orchestra.run.smithi022.stdout: 5 201750 10.99 MB/sec execute 314 sec latency 1043.873 ms 2024-03-21T12:42:15.145 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:15 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:42:15.145 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:15 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:42:15.145 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:15 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:42:15.145 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:15 smithi022 ceph-mon[67047]: pgmap v80: 97 pgs: 87 active+clean, 3 peering, 1 active+recovery_wait+degraded, 1 active+recovering+degraded, 5 active+recovery_wait+undersized+degraded+remapped; 337 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 2.8 MiB/s wr, 308 op/s; 157/1470 objects degraded (10.680%); 1.9 MiB/s, 28 objects/s recovering 2024-03-21T12:42:15.145 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:15 smithi022 ceph-mon[67047]: pgmap v81: 97 pgs: 87 active+clean, 3 peering, 1 active+recovery_wait+degraded, 1 active+recovering+degraded, 5 active+recovery_wait+undersized+degraded+remapped; 337 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 3.5 MiB/s wr, 385 op/s; 157/1470 objects degraded (10.680%); 2.3 MiB/s, 36 objects/s recovering 2024-03-21T12:42:15.145 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:15 smithi022 ceph-mon[67047]: osdmap e83: 8 total, 8 up, 8 in 2024-03-21T12:42:15.145 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:15 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:42:15.145 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:15 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:42:15.145 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:15 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:42:15.146 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:15 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:42:15.146 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:15 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:42:15.146 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:15 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:42:15.146 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:15 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-21T12:42:15.146 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:15 smithi022 ceph-mon[67047]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-21T12:42:15.146 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:15 smithi022 ceph-mon[67047]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-03-21T12:42:15.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:15 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:42:15.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:15 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:42:15.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:15 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:42:15.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:15 smithi189 ceph-mon[53160]: pgmap v80: 97 pgs: 87 active+clean, 3 peering, 1 active+recovery_wait+degraded, 1 active+recovering+degraded, 5 active+recovery_wait+undersized+degraded+remapped; 337 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 2.8 MiB/s wr, 308 op/s; 157/1470 objects degraded (10.680%); 1.9 MiB/s, 28 objects/s recovering 2024-03-21T12:42:15.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:15 smithi189 ceph-mon[53160]: pgmap v81: 97 pgs: 87 active+clean, 3 peering, 1 active+recovery_wait+degraded, 1 active+recovering+degraded, 5 active+recovery_wait+undersized+degraded+remapped; 337 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 3.5 MiB/s wr, 385 op/s; 157/1470 objects degraded (10.680%); 2.3 MiB/s, 36 objects/s recovering 2024-03-21T12:42:15.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:15 smithi189 ceph-mon[53160]: osdmap e83: 8 total, 8 up, 8 in 2024-03-21T12:42:15.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:15 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:42:15.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:15 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:42:15.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:15 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:42:15.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:15 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:42:15.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:15 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:42:15.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:15 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:42:15.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:15 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-21T12:42:15.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:15 smithi189 ceph-mon[53160]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-21T12:42:15.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:15 smithi189 ceph-mon[53160]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-03-21T12:42:15.875 INFO:teuthology.orchestra.run.smithi022.stdout: 5 202180 10.99 MB/sec execute 315 sec latency 610.580 ms 2024-03-21T12:42:16.179 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:16 smithi022 ceph-mon[67047]: osdmap e84: 8 total, 8 up, 8 in 2024-03-21T12:42:16.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:16 smithi189 ceph-mon[53160]: osdmap e84: 8 total, 8 up, 8 in 2024-03-21T12:42:16.875 INFO:teuthology.orchestra.run.smithi022.stdout: 5 202716 11.00 MB/sec execute 316 sec latency 874.347 ms 2024-03-21T12:42:17.183 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:17 smithi022 ceph-mon[67047]: pgmap v84: 97 pgs: 87 active+clean, 3 peering, 1 active+recovery_wait+degraded, 1 active+recovering+degraded, 5 active+recovery_wait+undersized+degraded+remapped; 337 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 157/1470 objects degraded (10.680%) 2024-03-21T12:42:17.183 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:17 smithi022 ceph-mon[67047]: osdmap e85: 8 total, 8 up, 8 in 2024-03-21T12:42:17.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:17 smithi189 ceph-mon[53160]: pgmap v84: 97 pgs: 87 active+clean, 3 peering, 1 active+recovery_wait+degraded, 1 active+recovering+degraded, 5 active+recovery_wait+undersized+degraded+remapped; 337 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 157/1470 objects degraded (10.680%) 2024-03-21T12:42:17.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:17 smithi189 ceph-mon[53160]: osdmap e85: 8 total, 8 up, 8 in 2024-03-21T12:42:17.875 INFO:teuthology.orchestra.run.smithi022.stdout: 5 203152 11.01 MB/sec execute 317 sec latency 365.973 ms 2024-03-21T12:42:18.194 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:18 smithi022 ceph-mon[67047]: osdmap e86: 8 total, 8 up, 8 in 2024-03-21T12:42:18.195 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:18 smithi022 ceph-mon[67047]: osdmap e87: 8 total, 8 up, 8 in 2024-03-21T12:42:18.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:18 smithi189 ceph-mon[53160]: osdmap e86: 8 total, 8 up, 8 in 2024-03-21T12:42:18.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:18 smithi189 ceph-mon[53160]: osdmap e87: 8 total, 8 up, 8 in 2024-03-21T12:42:18.875 INFO:teuthology.orchestra.run.smithi022.stdout: 5 203596 11.02 MB/sec execute 318 sec latency 357.007 ms 2024-03-21T12:42:19.203 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:19 smithi022 ceph-mon[67047]: pgmap v87: 97 pgs: 1 peering, 95 active+clean, 1 active+recovering+undersized+degraded+remapped; 343 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 7.2 KiB/s rd, 6.8 MiB/s wr, 700 op/s; 1024 KiB/s, 42 objects/s recovering 2024-03-21T12:42:19.204 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:19 smithi022 ceph-mon[67047]: Health check update: Degraded data redundancy: 1 pg degraded (PG_DEGRADED) 2024-03-21T12:42:19.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:19 smithi189 ceph-mon[53160]: pgmap v87: 97 pgs: 1 peering, 95 active+clean, 1 active+recovering+undersized+degraded+remapped; 343 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 7.2 KiB/s rd, 6.8 MiB/s wr, 700 op/s; 1024 KiB/s, 42 objects/s recovering 2024-03-21T12:42:19.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:19 smithi189 ceph-mon[53160]: Health check update: Degraded data redundancy: 1 pg degraded (PG_DEGRADED) 2024-03-21T12:42:19.875 INFO:teuthology.orchestra.run.smithi022.stdout: 5 203994 11.01 MB/sec execute 319 sec latency 50.721 ms 2024-03-21T12:42:20.486 INFO:teuthology.orchestra.run.smithi022.stdout:true 2024-03-21T12:42:20.875 INFO:teuthology.orchestra.run.smithi022.stdout: 5 204387 11.01 MB/sec execute 320 sec latency 35.161 ms 2024-03-21T12:42:21.248 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:21 smithi022 ceph-mon[67047]: pgmap v89: 97 pgs: 1 peering, 95 active+clean, 1 active+recovering+undersized+degraded+remapped; 343 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 5.8 KiB/s rd, 5.5 MiB/s wr, 564 op/s; 825 KiB/s, 34 objects/s recovering 2024-03-21T12:42:21.248 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:21 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:42:21.248 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:21 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:42:21.448 INFO:teuthology.orchestra.run.smithi022.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-21T12:42:21.448 INFO:teuthology.orchestra.run.smithi022.stdout:alertmanager.smithi022 smithi022 *:9093,9094 running (4m) 13s ago 13m 24.1M - 0.23.0 ba2b418f427c 14eb2a3ca318 2024-03-21T12:42:21.448 INFO:teuthology.orchestra.run.smithi022.stdout:crash.smithi022 smithi022 running (61s) 13s ago 13m 6660k - 19.0.0-2267-g960fbbe4 19e12293512f 356b2ddb609a 2024-03-21T12:42:21.448 INFO:teuthology.orchestra.run.smithi022.stdout:crash.smithi189 smithi189 running (54s) 49s ago 13m 6660k - 19.0.0-2267-g960fbbe4 19e12293512f 90ee837f0500 2024-03-21T12:42:21.448 INFO:teuthology.orchestra.run.smithi022.stdout:grafana.smithi022 smithi022 *:3000 error 13s ago 13m - - 2024-03-21T12:42:21.448 INFO:teuthology.orchestra.run.smithi022.stdout:mds.foofs.smithi022.bcnxho smithi022 running (7m) 13s ago 7m 38.0M - 17.2.0 e1d6a67b021e db26b67b8ee7 2024-03-21T12:42:21.448 INFO:teuthology.orchestra.run.smithi022.stdout:mds.foofs.smithi189.lluvko smithi189 running (7m) 49s ago 7m 16.9M - 17.2.0 e1d6a67b021e cf153aff1387 2024-03-21T12:42:21.448 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.smithi022.fektzs smithi022 *:8443,9283,8765 running (3m) 13s ago 14m 501M - 19.0.0-2267-g960fbbe4 19e12293512f d57cebd306b3 2024-03-21T12:42:21.448 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.smithi189.cnxzvu smithi189 *:8443,9283,8765 running (2m) 49s ago 13m 450M - 19.0.0-2267-g960fbbe4 19e12293512f 41663955c56f 2024-03-21T12:42:21.448 INFO:teuthology.orchestra.run.smithi022.stdout:mon.smithi022 smithi022 running (2m) 13s ago 15m 46.1M 2048M 19.0.0-2267-g960fbbe4 19e12293512f c620c9071b2c 2024-03-21T12:42:21.448 INFO:teuthology.orchestra.run.smithi022.stdout:mon.smithi189 smithi189 running (117s) 49s ago 12m 35.8M 2048M 19.0.0-2267-g960fbbe4 19e12293512f b353af3d8f3b 2024-03-21T12:42:21.448 INFO:teuthology.orchestra.run.smithi022.stdout:nfs.foo.0.0.smithi022.ynhbbq smithi022 *:2049 running (7m) 13s ago 7m 258M - 4.0 e1d6a67b021e 33e48e64c60b 2024-03-21T12:42:21.449 INFO:teuthology.orchestra.run.smithi022.stdout:nfs.foo.1.0.smithi189.yzsqml smithi189 *:2049 running (7m) 49s ago 7m 85.4M - 4.0 e1d6a67b021e 48fab2cbdde2 2024-03-21T12:42:21.449 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.smithi022 smithi022 *:9100 running (4m) 13s ago 13m 17.3M - 1.3.1 1dbe0e931976 33287f803f4f 2024-03-21T12:42:21.449 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.smithi189 smithi189 *:9100 running (4m) 49s ago 12m 17.5M - 1.3.1 1dbe0e931976 b51d872e438a 2024-03-21T12:42:21.449 INFO:teuthology.orchestra.run.smithi022.stdout:osd.0 smithi022 running (23s) 13s ago 12m 12.0M 1937M 19.0.0-2267-g960fbbe4 19e12293512f 23a85645eeea 2024-03-21T12:42:21.449 INFO:teuthology.orchestra.run.smithi022.stdout:osd.1 smithi022 running (11m) 13s ago 11m 329M 1937M 17.2.0 e1d6a67b021e 68165c5d09cb 2024-03-21T12:42:21.449 INFO:teuthology.orchestra.run.smithi022.stdout:osd.2 smithi022 running (11m) 13s ago 11m 487M 1937M 17.2.0 e1d6a67b021e fbec9bef8433 2024-03-21T12:42:21.449 INFO:teuthology.orchestra.run.smithi022.stdout:osd.3 smithi022 running (10m) 13s ago 10m 602M 1937M 17.2.0 e1d6a67b021e b1462ffc1b9e 2024-03-21T12:42:21.449 INFO:teuthology.orchestra.run.smithi022.stdout:osd.4 smithi189 running (10m) 49s ago 10m 560M 2704M 17.2.0 e1d6a67b021e 1bedc4eb02ed 2024-03-21T12:42:21.449 INFO:teuthology.orchestra.run.smithi022.stdout:osd.5 smithi189 running (10m) 49s ago 10m 407M 2704M 17.2.0 e1d6a67b021e 25fbab9786a5 2024-03-21T12:42:21.449 INFO:teuthology.orchestra.run.smithi022.stdout:osd.6 smithi189 running (9m) 49s ago 9m 578M 2704M 17.2.0 e1d6a67b021e 0c192fcfd65b 2024-03-21T12:42:21.449 INFO:teuthology.orchestra.run.smithi022.stdout:osd.7 smithi189 running (9m) 49s ago 9m 528M 2704M 17.2.0 e1d6a67b021e 3d0723e1f863 2024-03-21T12:42:21.449 INFO:teuthology.orchestra.run.smithi022.stdout:prometheus.smithi022 smithi022 *:9095 running (2m) 13s ago 12m 73.1M - 2.33.4 514e6a882f6e 8171bbe41d2c 2024-03-21T12:42:21.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:21 smithi189 ceph-mon[53160]: pgmap v89: 97 pgs: 1 peering, 95 active+clean, 1 active+recovering+undersized+degraded+remapped; 343 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 5.8 KiB/s rd, 5.5 MiB/s wr, 564 op/s; 825 KiB/s, 34 objects/s recovering 2024-03-21T12:42:21.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:21 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:42:21.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:21 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:42:21.875 INFO:teuthology.orchestra.run.smithi022.stdout: 5 204679 11.00 MB/sec execute 321 sec latency 52.828 ms 2024-03-21T12:42:22.059 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-03-21T12:42:22.060 INFO:teuthology.orchestra.run.smithi022.stdout: "mon": { 2024-03-21T12:42:22.060 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 2 2024-03-21T12:42:22.060 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:42:22.060 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr": { 2024-03-21T12:42:22.060 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 2 2024-03-21T12:42:22.060 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:42:22.060 INFO:teuthology.orchestra.run.smithi022.stdout: "osd": { 2024-03-21T12:42:22.060 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-03-21T12:42:22.060 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 1 2024-03-21T12:42:22.060 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:42:22.060 INFO:teuthology.orchestra.run.smithi022.stdout: "mds": { 2024-03-21T12:42:22.061 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-21T12:42:22.061 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:42:22.061 INFO:teuthology.orchestra.run.smithi022.stdout: "overall": { 2024-03-21T12:42:22.061 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 9, 2024-03-21T12:42:22.061 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 5 2024-03-21T12:42:22.061 INFO:teuthology.orchestra.run.smithi022.stdout: } 2024-03-21T12:42:22.061 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-03-21T12:42:22.262 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:22 smithi022 ceph-mon[67047]: from='client.34226 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:42:22.263 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:22 smithi022 ceph-mon[67047]: from='client.34230 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:42:22.263 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:22 smithi022 ceph-mon[67047]: from='client.? 172.21.15.22:0/3924864894' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:42:22.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:22 smithi189 ceph-mon[53160]: from='client.34226 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:42:22.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:22 smithi189 ceph-mon[53160]: from='client.34230 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:42:22.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:22 smithi189 ceph-mon[53160]: from='client.? 172.21.15.22:0/3924864894' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:42:22.612 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-03-21T12:42:22.612 INFO:teuthology.orchestra.run.smithi022.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93", 2024-03-21T12:42:22.612 INFO:teuthology.orchestra.run.smithi022.stdout: "in_progress": true, 2024-03-21T12:42:22.612 INFO:teuthology.orchestra.run.smithi022.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-21T12:42:22.612 INFO:teuthology.orchestra.run.smithi022.stdout: "services_complete": [ 2024-03-21T12:42:22.612 INFO:teuthology.orchestra.run.smithi022.stdout: "mon", 2024-03-21T12:42:22.612 INFO:teuthology.orchestra.run.smithi022.stdout: "crash", 2024-03-21T12:42:22.613 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr" 2024-03-21T12:42:22.613 INFO:teuthology.orchestra.run.smithi022.stdout: ], 2024-03-21T12:42:22.613 INFO:teuthology.orchestra.run.smithi022.stdout: "progress": "7/23 daemons upgraded", 2024-03-21T12:42:22.613 INFO:teuthology.orchestra.run.smithi022.stdout: "message": "Currently upgrading osd daemons", 2024-03-21T12:42:22.613 INFO:teuthology.orchestra.run.smithi022.stdout: "is_paused": false 2024-03-21T12:42:22.613 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-03-21T12:42:22.875 INFO:teuthology.orchestra.run.smithi022.stdout: 5 205150 11.01 MB/sec execute 322 sec latency 44.679 ms 2024-03-21T12:42:23.264 INFO:teuthology.orchestra.run.smithi022.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-03-21T12:42:23.264 INFO:teuthology.orchestra.run.smithi022.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-21T12:42:23.264 INFO:teuthology.orchestra.run.smithi022.stdout: daemon grafana.smithi022 on smithi022 is in error state 2024-03-21T12:42:23.283 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:23 smithi022 ceph-mon[67047]: from='client.34234 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:42:23.283 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:23 smithi022 ceph-mon[67047]: pgmap v90: 97 pgs: 97 active+clean; 345 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 12 KiB/s rd, 9.8 MiB/s wr, 962 op/s; 683 KiB/s, 28 objects/s recovering 2024-03-21T12:42:23.283 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:23 smithi022 ceph-mon[67047]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1 pg degraded) 2024-03-21T12:42:23.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:23 smithi189 ceph-mon[53160]: from='client.34234 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:42:23.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:23 smithi189 ceph-mon[53160]: pgmap v90: 97 pgs: 97 active+clean; 345 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 12 KiB/s rd, 9.8 MiB/s wr, 962 op/s; 683 KiB/s, 28 objects/s recovering 2024-03-21T12:42:23.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:23 smithi189 ceph-mon[53160]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1 pg degraded) 2024-03-21T12:42:23.875 INFO:teuthology.orchestra.run.smithi022.stdout: 5 205673 11.01 MB/sec execute 323 sec latency 27.335 ms 2024-03-21T12:42:24.299 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:24 smithi022 ceph-mon[67047]: from='client.34242 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:42:24.299 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:24 smithi022 ceph-mon[67047]: from='client.? 172.21.15.22:0/2183730190' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-21T12:42:24.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:24 smithi189 ceph-mon[53160]: from='client.34242 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:42:24.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:24 smithi189 ceph-mon[53160]: from='client.? 172.21.15.22:0/2183730190' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-21T12:42:24.875 INFO:teuthology.orchestra.run.smithi022.stdout: 5 206048 11.01 MB/sec execute 324 sec latency 38.855 ms 2024-03-21T12:42:25.325 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:25 smithi022 ceph-mon[67047]: pgmap v91: 97 pgs: 97 active+clean; 345 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 9.0 KiB/s rd, 7.4 MiB/s wr, 725 op/s; 514 KiB/s, 21 objects/s recovering 2024-03-21T12:42:25.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:25 smithi189 ceph-mon[53160]: pgmap v91: 97 pgs: 97 active+clean; 345 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 9.0 KiB/s rd, 7.4 MiB/s wr, 725 op/s; 514 KiB/s, 21 objects/s recovering 2024-03-21T12:42:25.875 INFO:teuthology.orchestra.run.smithi022.stdout: 5 206469 11.01 MB/sec execute 325 sec latency 47.740 ms 2024-03-21T12:42:26.875 INFO:teuthology.orchestra.run.smithi022.stdout: 5 206895 11.01 MB/sec execute 326 sec latency 37.688 ms 2024-03-21T12:42:27.357 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:27 smithi022 ceph-mon[67047]: pgmap v92: 97 pgs: 97 active+clean; 345 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4.8 KiB/s rd, 3.5 MiB/s wr, 331 op/s 2024-03-21T12:42:27.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:27 smithi189 ceph-mon[53160]: pgmap v92: 97 pgs: 97 active+clean; 345 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4.8 KiB/s rd, 3.5 MiB/s wr, 331 op/s 2024-03-21T12:42:27.875 INFO:teuthology.orchestra.run.smithi022.stdout: 5 207320 11.02 MB/sec execute 327 sec latency 47.270 ms 2024-03-21T12:42:28.875 INFO:teuthology.orchestra.run.smithi022.stdout: 5 207792 11.02 MB/sec execute 328 sec latency 28.344 ms 2024-03-21T12:42:29.380 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:29 smithi022 ceph-mon[67047]: pgmap v93: 97 pgs: 97 active+clean; 346 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 9.0 KiB/s rd, 6.2 MiB/s wr, 609 op/s 2024-03-21T12:42:29.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:29 smithi189 ceph-mon[53160]: pgmap v93: 97 pgs: 97 active+clean; 346 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 9.0 KiB/s rd, 6.2 MiB/s wr, 609 op/s 2024-03-21T12:42:29.875 INFO:teuthology.orchestra.run.smithi022.stdout: 5 208056 11.01 MB/sec execute 329 sec latency 49.285 ms 2024-03-21T12:42:30.402 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:30 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-21T12:42:30.402 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:30 smithi022 ceph-mon[67047]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-21T12:42:30.403 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:30 smithi022 ceph-mon[67047]: Upgrade: osd.1 is safe to restart 2024-03-21T12:42:30.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:30 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-21T12:42:30.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:30 smithi189 ceph-mon[53160]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-21T12:42:30.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:30 smithi189 ceph-mon[53160]: Upgrade: osd.1 is safe to restart 2024-03-21T12:42:30.875 INFO:teuthology.orchestra.run.smithi022.stdout: 5 208263 10.99 MB/sec execute 330 sec latency 63.523 ms 2024-03-21T12:42:31.200 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:31 smithi022 ceph-mon[67047]: pgmap v94: 97 pgs: 97 active+clean; 346 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.2 MiB/s wr, 509 op/s 2024-03-21T12:42:31.201 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:31 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:42:31.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:31 smithi189 ceph-mon[53160]: pgmap v94: 97 pgs: 97 active+clean; 346 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.2 MiB/s wr, 509 op/s 2024-03-21T12:42:31.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:31 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:42:31.875 INFO:teuthology.orchestra.run.smithi022.stdout: 5 208540 10.99 MB/sec execute 331 sec latency 71.399 ms 2024-03-21T12:42:32.763 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:32 smithi022 ceph-mon[67047]: Upgrade: Updating osd.1 2024-03-21T12:42:32.764 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:32 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:42:32.764 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:32 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-03-21T12:42:32.764 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:32 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:42:32.764 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:32 smithi022 ceph-mon[67047]: Deploying daemon osd.1 on smithi022 2024-03-21T12:42:32.764 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:32 smithi022 ceph-mon[67047]: pgmap v95: 97 pgs: 97 active+clean; 342 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 7.3 MiB/s wr, 723 op/s 2024-03-21T12:42:32.875 INFO:teuthology.orchestra.run.smithi022.stdout: 5 208795 10.97 MB/sec execute 332 sec latency 67.864 ms 2024-03-21T12:42:33.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:32 smithi189 ceph-mon[53160]: Upgrade: Updating osd.1 2024-03-21T12:42:33.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:32 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:42:33.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:32 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-03-21T12:42:33.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:32 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:42:33.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:32 smithi189 ceph-mon[53160]: Deploying daemon osd.1 on smithi022 2024-03-21T12:42:33.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:32 smithi189 ceph-mon[53160]: pgmap v95: 97 pgs: 97 active+clean; 342 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 7.3 MiB/s wr, 723 op/s 2024-03-21T12:42:33.875 INFO:teuthology.orchestra.run.smithi022.stdout: 5 209101 10.96 MB/sec execute 333 sec latency 72.068 ms 2024-03-21T12:42:34.876 INFO:teuthology.orchestra.run.smithi022.stdout: 5 209346 10.95 MB/sec execute 334 sec latency 47.496 ms 2024-03-21T12:42:35.325 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:35 smithi022 ceph-mon[67047]: pgmap v96: 97 pgs: 97 active+clean; 342 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.7 MiB/s wr, 476 op/s 2024-03-21T12:42:35.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:35 smithi189 ceph-mon[53160]: pgmap v96: 97 pgs: 97 active+clean; 342 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.7 MiB/s wr, 476 op/s 2024-03-21T12:42:35.876 INFO:teuthology.orchestra.run.smithi022.stdout: 5 209607 10.93 MB/sec execute 335 sec latency 55.941 ms 2024-03-21T12:42:36.876 INFO:teuthology.orchestra.run.smithi022.stdout: 5 209936 10.94 MB/sec execute 336 sec latency 47.400 ms 2024-03-21T12:42:36.889 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:36 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:42:36.890 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:36 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:42:36.890 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:36 smithi022 ceph-mon[67047]: pgmap v97: 97 pgs: 97 active+clean; 342 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.7 MiB/s wr, 476 op/s 2024-03-21T12:42:36.890 INFO:journalctl@ceph.osd.1.smithi022.stdout:Mar 21 12:42:36 smithi022 systemd[1]: Stopping Ceph osd.1 for 3fc7f0f2-e77e-11ee-95cd-87774f69a715... 2024-03-21T12:42:37.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:36 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:42:37.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:36 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:42:37.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:36 smithi189 ceph-mon[53160]: pgmap v97: 97 pgs: 97 active+clean; 342 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.7 MiB/s wr, 476 op/s 2024-03-21T12:42:37.149 INFO:journalctl@ceph.osd.1.smithi022.stdout:Mar 21 12:42:37 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-1[38048]: 2024-03-21T12:42:37.131+0000 7f0cd8b57700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-21T12:42:37.150 INFO:journalctl@ceph.osd.1.smithi022.stdout:Mar 21 12:42:37 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-1[38048]: 2024-03-21T12:42:37.131+0000 7f0cd8b57700 -1 osd.1 87 *** Got signal Terminated *** 2024-03-21T12:42:37.150 INFO:journalctl@ceph.osd.1.smithi022.stdout:Mar 21 12:42:37 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-1[38048]: 2024-03-21T12:42:37.131+0000 7f0cd8b57700 -1 osd.1 87 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-21T12:42:37.877 INFO:teuthology.orchestra.run.smithi022.stdout: 5 210230 10.92 MB/sec execute 337 sec latency 123.219 ms 2024-03-21T12:42:37.902 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:37 smithi022 ceph-mon[67047]: osd.1 marked itself down and dead 2024-03-21T12:42:38.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:37 smithi189 ceph-mon[53160]: osd.1 marked itself down and dead 2024-03-21T12:42:38.659 INFO:journalctl@ceph.osd.1.smithi022.stdout:Mar 21 12:42:38 smithi022 podman[75177]: 2024-03-21 12:42:38.386513073 +0000 UTC m=+1.377328836 container died 68165c5d09cb8f22c3856189d2d60fd601c1edc09659c467de54691afb33cf29 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-1, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, release=754, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, name=centos-stream, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, RELEASE=HEAD, vcs-type=git, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, distribution-scope=public) 2024-03-21T12:42:38.876 INFO:teuthology.orchestra.run.smithi022.stdout: 5 210302 10.89 MB/sec execute 338 sec latency 1009.715 ms 2024-03-21T12:42:38.910 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:38 smithi022 ceph-mon[67047]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-21T12:42:38.910 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:38 smithi022 ceph-mon[67047]: osdmap e88: 8 total, 7 up, 8 in 2024-03-21T12:42:38.910 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:38 smithi022 ceph-mon[67047]: pgmap v99: 97 pgs: 13 stale+active+clean, 84 active+clean; 343 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.7 MiB/s wr, 462 op/s 2024-03-21T12:42:38.910 INFO:journalctl@ceph.osd.1.smithi022.stdout:Mar 21 12:42:38 smithi022 podman[75177]: 2024-03-21 12:42:38.762574499 +0000 UTC m=+1.753390269 container cleanup 68165c5d09cb8f22c3856189d2d60fd601c1edc09659c467de54691afb33cf29 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-1, RELEASE=HEAD, architecture=x86_64, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, vcs-type=git, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, maintainer=Guillaume Abrioux , distribution-scope=public, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-03-21T12:42:38.910 INFO:journalctl@ceph.osd.1.smithi022.stdout:Mar 21 12:42:38 smithi022 bash[75177]: ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-1 2024-03-21T12:42:39.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:38 smithi189 ceph-mon[53160]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-21T12:42:39.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:38 smithi189 ceph-mon[53160]: osdmap e88: 8 total, 7 up, 8 in 2024-03-21T12:42:39.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:38 smithi189 ceph-mon[53160]: pgmap v99: 97 pgs: 13 stale+active+clean, 84 active+clean; 343 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.7 MiB/s wr, 462 op/s 2024-03-21T12:42:39.165 INFO:journalctl@ceph.osd.1.smithi022.stdout:Mar 21 12:42:39 smithi022 podman[75193]: 2024-03-21 12:42:39.131532018 +0000 UTC m=+0.738425755 container remove 68165c5d09cb8f22c3856189d2d60fd601c1edc09659c467de54691afb33cf29 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-1, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, release=754, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, ceph=True, distribution-scope=public, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-03-21T12:42:39.682 INFO:journalctl@ceph.osd.1.smithi022.stdout:Mar 21 12:42:39 smithi022 podman[75311]: 2024-03-21 12:42:39.475281256 +0000 UTC m=+0.030435819 image pull 19e12293512ff7978c8adb5f72a829060dd47f07bfd6903b6b3178d2ebc29e4e quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93 2024-03-21T12:42:39.876 INFO:teuthology.orchestra.run.smithi022.stdout: 5 210700 10.90 MB/sec execute 339 sec latency 50.038 ms 2024-03-21T12:42:39.940 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:39 smithi022 ceph-mon[67047]: osdmap e89: 8 total, 7 up, 8 in 2024-03-21T12:42:39.941 INFO:journalctl@ceph.osd.1.smithi022.stdout:Mar 21 12:42:39 smithi022 podman[75311]: 2024-03-21 12:42:39.775061054 +0000 UTC m=+0.330215589 container create 108c8a067b49878b88b7c75d672ed6063d958e471262e51a15af26d8191f64b8 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-1-deactivate, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.build-date=20240319, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-21T12:42:40.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:39 smithi189 ceph-mon[53160]: osdmap e89: 8 total, 7 up, 8 in 2024-03-21T12:42:40.192 INFO:journalctl@ceph.osd.1.smithi022.stdout:Mar 21 12:42:40 smithi022 podman[75311]: 2024-03-21 12:42:40.18577204 +0000 UTC m=+0.740926601 container init 108c8a067b49878b88b7c75d672ed6063d958e471262e51a15af26d8191f64b8 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-1-deactivate, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240319, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_CLEAN=True, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True) 2024-03-21T12:42:40.445 INFO:journalctl@ceph.osd.1.smithi022.stdout:Mar 21 12:42:40 smithi022 podman[75311]: 2024-03-21 12:42:40.194827648 +0000 UTC m=+0.749982185 container start 108c8a067b49878b88b7c75d672ed6063d958e471262e51a15af26d8191f64b8 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-1-deactivate, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240319, GIT_CLEAN=True, ceph=True, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-21T12:42:40.445 INFO:journalctl@ceph.osd.1.smithi022.stdout:Mar 21 12:42:40 smithi022 podman[75311]: 2024-03-21 12:42:40.39005199 +0000 UTC m=+0.945206544 container attach 108c8a067b49878b88b7c75d672ed6063d958e471262e51a15af26d8191f64b8 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-1-deactivate, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.build-date=20240319, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-21T12:42:40.696 INFO:journalctl@ceph.osd.1.smithi022.stdout:Mar 21 12:42:40 smithi022 podman[75311]: 2024-03-21 12:42:40.643216683 +0000 UTC m=+1.198371226 container died 108c8a067b49878b88b7c75d672ed6063d958e471262e51a15af26d8191f64b8 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-1-deactivate, GIT_BRANCH=HEAD, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240319, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5) 2024-03-21T12:42:40.877 INFO:teuthology.orchestra.run.smithi022.stdout: 5 211077 10.90 MB/sec execute 340 sec latency 46.263 ms 2024-03-21T12:42:40.946 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:40 smithi022 ceph-mon[67047]: pgmap v101: 97 pgs: 13 stale+active+clean, 84 active+clean; 343 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4.0 KiB/s rd, 2.5 MiB/s wr, 254 op/s 2024-03-21T12:42:41.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:40 smithi189 ceph-mon[53160]: pgmap v101: 97 pgs: 13 stale+active+clean, 84 active+clean; 343 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4.0 KiB/s rd, 2.5 MiB/s wr, 254 op/s 2024-03-21T12:42:41.709 INFO:journalctl@ceph.osd.1.smithi022.stdout:Mar 21 12:42:41 smithi022 podman[75336]: 2024-03-21 12:42:41.587440155 +0000 UTC m=+0.938032295 container remove 108c8a067b49878b88b7c75d672ed6063d958e471262e51a15af26d8191f64b8 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-1-deactivate, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240319, ceph=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, CEPH_POINT_RELEASE=, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux ) 2024-03-21T12:42:41.709 INFO:journalctl@ceph.osd.1.smithi022.stdout:Mar 21 12:42:41 smithi022 systemd[1]: ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715@osd.1.service: Deactivated successfully. 2024-03-21T12:42:41.709 INFO:journalctl@ceph.osd.1.smithi022.stdout:Mar 21 12:42:41 smithi022 systemd[1]: Stopped Ceph osd.1 for 3fc7f0f2-e77e-11ee-95cd-87774f69a715. 2024-03-21T12:42:41.709 INFO:journalctl@ceph.osd.1.smithi022.stdout:Mar 21 12:42:41 smithi022 systemd[1]: ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715@osd.1.service: Consumed 3min 41.747s CPU time. 2024-03-21T12:42:41.877 INFO:teuthology.orchestra.run.smithi022.stdout: 5 211501 10.89 MB/sec execute 341 sec latency 32.781 ms 2024-03-21T12:42:41.966 INFO:journalctl@ceph.osd.1.smithi022.stdout:Mar 21 12:42:41 smithi022 systemd[1]: Starting Ceph osd.1 for 3fc7f0f2-e77e-11ee-95cd-87774f69a715... 2024-03-21T12:42:42.218 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:42 smithi022 ceph-mon[67047]: Health check failed: Degraded data redundancy: 192/1701 objects degraded (11.287%), 24 pgs degraded (PG_DEGRADED) 2024-03-21T12:42:42.219 INFO:journalctl@ceph.osd.1.smithi022.stdout:Mar 21 12:42:42 smithi022 podman[75429]: 2024-03-21 12:42:42.098845213 +0000 UTC m=+0.046827380 image pull 19e12293512ff7978c8adb5f72a829060dd47f07bfd6903b6b3178d2ebc29e4e quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93 2024-03-21T12:42:42.219 INFO:journalctl@ceph.osd.1.smithi022.stdout:Mar 21 12:42:42 smithi022 podman[75429]: 2024-03-21 12:42:42.214055181 +0000 UTC m=+0.162037329 container create 51cf6d3295613f6139ff138cdda86a6a60df8624f25bc5d8183357dad708eba0 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-1-activate, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.license=GPLv2, ceph=True, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, GIT_CLEAN=True, org.label-schema.build-date=20240319, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-21T12:42:42.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:42 smithi189 ceph-mon[53160]: Health check failed: Degraded data redundancy: 192/1701 objects degraded (11.287%), 24 pgs degraded (PG_DEGRADED) 2024-03-21T12:42:42.877 INFO:teuthology.orchestra.run.smithi022.stdout: 5 211870 10.90 MB/sec execute 342 sec latency 35.677 ms 2024-03-21T12:42:42.880 INFO:journalctl@ceph.osd.1.smithi022.stdout:Mar 21 12:42:42 smithi022 podman[75429]: 2024-03-21 12:42:42.854283262 +0000 UTC m=+0.802265420 container init 51cf6d3295613f6139ff138cdda86a6a60df8624f25bc5d8183357dad708eba0 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-1-activate, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, maintainer=Guillaume Abrioux , RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240319, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-03-21T12:42:42.880 INFO:journalctl@ceph.osd.1.smithi022.stdout:Mar 21 12:42:42 smithi022 podman[75429]: 2024-03-21 12:42:42.867486164 +0000 UTC m=+0.815468307 container start 51cf6d3295613f6139ff138cdda86a6a60df8624f25bc5d8183357dad708eba0 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-1-activate, io.buildah.version=1.33.5, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, org.label-schema.build-date=20240319, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-03-21T12:42:43.239 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:43 smithi022 ceph-mon[67047]: pgmap v102: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 348 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.4 MiB/s wr, 533 op/s; 192/1701 objects degraded (11.287%) 2024-03-21T12:42:43.240 INFO:journalctl@ceph.osd.1.smithi022.stdout:Mar 21 12:42:43 smithi022 podman[75429]: 2024-03-21 12:42:43.020782698 +0000 UTC m=+0.968764840 container attach 51cf6d3295613f6139ff138cdda86a6a60df8624f25bc5d8183357dad708eba0 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-1-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, org.label-schema.build-date=20240319, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.schema-version=1.0) 2024-03-21T12:42:43.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:43 smithi189 ceph-mon[53160]: pgmap v102: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 348 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.4 MiB/s wr, 533 op/s; 192/1701 objects degraded (11.287%) 2024-03-21T12:42:43.748 INFO:journalctl@ceph.osd.1.smithi022.stdout:Mar 21 12:42:43 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-1-activate[75444]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-03-21T12:42:43.748 INFO:journalctl@ceph.osd.1.smithi022.stdout:Mar 21 12:42:43 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-1-activate[75444]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-03-21T12:42:43.749 INFO:journalctl@ceph.osd.1.smithi022.stdout:Mar 21 12:42:43 smithi022 bash[75429]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-03-21T12:42:43.749 INFO:journalctl@ceph.osd.1.smithi022.stdout:Mar 21 12:42:43 smithi022 bash[75429]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-03-21T12:42:43.749 INFO:journalctl@ceph.osd.1.smithi022.stdout:Mar 21 12:42:43 smithi022 bash[75429]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-03-21T12:42:43.749 INFO:journalctl@ceph.osd.1.smithi022.stdout:Mar 21 12:42:43 smithi022 bash[75429]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-03-21T12:42:43.749 INFO:journalctl@ceph.osd.1.smithi022.stdout:Mar 21 12:42:43 smithi022 bash[75429]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-03-21T12:42:43.749 INFO:journalctl@ceph.osd.1.smithi022.stdout:Mar 21 12:42:43 smithi022 bash[75429]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-03-21T12:42:43.749 INFO:journalctl@ceph.osd.1.smithi022.stdout:Mar 21 12:42:43 smithi022 bash[75429]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-03-21T12:42:43.749 INFO:journalctl@ceph.osd.1.smithi022.stdout:Mar 21 12:42:43 smithi022 bash[75429]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-03-21T12:42:43.749 INFO:journalctl@ceph.osd.1.smithi022.stdout:Mar 21 12:42:43 smithi022 bash[75429]: --> ceph-volume raw activate successful for osd ID: 1 2024-03-21T12:42:43.749 INFO:journalctl@ceph.osd.1.smithi022.stdout:Mar 21 12:42:43 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-1-activate[75444]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-03-21T12:42:43.749 INFO:journalctl@ceph.osd.1.smithi022.stdout:Mar 21 12:42:43 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-1-activate[75444]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-03-21T12:42:43.749 INFO:journalctl@ceph.osd.1.smithi022.stdout:Mar 21 12:42:43 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-1-activate[75444]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-03-21T12:42:43.749 INFO:journalctl@ceph.osd.1.smithi022.stdout:Mar 21 12:42:43 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-1-activate[75444]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-03-21T12:42:43.749 INFO:journalctl@ceph.osd.1.smithi022.stdout:Mar 21 12:42:43 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-1-activate[75444]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-03-21T12:42:43.749 INFO:journalctl@ceph.osd.1.smithi022.stdout:Mar 21 12:42:43 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-1-activate[75444]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-03-21T12:42:43.750 INFO:journalctl@ceph.osd.1.smithi022.stdout:Mar 21 12:42:43 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-1-activate[75444]: --> ceph-volume raw activate successful for osd ID: 1 2024-03-21T12:42:43.750 INFO:journalctl@ceph.osd.1.smithi022.stdout:Mar 21 12:42:43 smithi022 podman[75575]: 2024-03-21 12:42:43.581442068 +0000 UTC m=+0.026116491 container died 51cf6d3295613f6139ff138cdda86a6a60df8624f25bc5d8183357dad708eba0 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-1-activate, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.build-date=20240319, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, GIT_BRANCH=HEAD) 2024-03-21T12:42:43.876 INFO:teuthology.orchestra.run.smithi022.stdout: 5 212236 10.89 MB/sec execute 343 sec latency 68.591 ms 2024-03-21T12:42:44.754 INFO:journalctl@ceph.osd.1.smithi022.stdout:Mar 21 12:42:44 smithi022 podman[75575]: 2024-03-21 12:42:44.636708218 +0000 UTC m=+1.081382642 container remove 51cf6d3295613f6139ff138cdda86a6a60df8624f25bc5d8183357dad708eba0 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-1-activate, org.label-schema.build-date=20240319, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, GIT_CLEAN=True, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0) 2024-03-21T12:42:44.877 INFO:teuthology.orchestra.run.smithi022.stdout: 5 212665 10.89 MB/sec execute 344 sec latency 43.013 ms 2024-03-21T12:42:45.006 INFO:journalctl@ceph.osd.1.smithi022.stdout:Mar 21 12:42:44 smithi022 podman[75632]: 2024-03-21 12:42:44.80820576 +0000 UTC m=+0.029303714 image pull 19e12293512ff7978c8adb5f72a829060dd47f07bfd6903b6b3178d2ebc29e4e quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93 2024-03-21T12:42:45.006 INFO:journalctl@ceph.osd.1.smithi022.stdout:Mar 21 12:42:44 smithi022 podman[75632]: 2024-03-21 12:42:44.999450663 +0000 UTC m=+0.220548606 container create 0f90e70bbf0a14657e3b081b7429efc76d72f983daf098524e30a26f32010134 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-1, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240319, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-03-21T12:42:45.261 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:45 smithi022 ceph-mon[67047]: pgmap v103: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 348 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.4 MiB/s wr, 533 op/s; 192/1701 objects degraded (11.287%) 2024-03-21T12:42:45.520 INFO:journalctl@ceph.osd.1.smithi022.stdout:Mar 21 12:42:45 smithi022 podman[75632]: 2024-03-21 12:42:45.392675669 +0000 UTC m=+0.613773613 container init 0f90e70bbf0a14657e3b081b7429efc76d72f983daf098524e30a26f32010134 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-1, org.label-schema.build-date=20240319, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-21T12:42:45.520 INFO:journalctl@ceph.osd.1.smithi022.stdout:Mar 21 12:42:45 smithi022 podman[75632]: 2024-03-21 12:42:45.397730139 +0000 UTC m=+0.618828085 container start 0f90e70bbf0a14657e3b081b7429efc76d72f983daf098524e30a26f32010134 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-1, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.build-date=20240319, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True) 2024-03-21T12:42:45.520 INFO:journalctl@ceph.osd.1.smithi022.stdout:Mar 21 12:42:45 smithi022 ceph-osd[75650]: -- 172.21.15.22:0/3856569167 <== mon.0 v2:172.21.15.22:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55a2fa6b1680 con 0x55a2fa67ac00 2024-03-21T12:42:45.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:45 smithi189 ceph-mon[53160]: pgmap v103: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 348 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.4 MiB/s wr, 533 op/s; 192/1701 objects degraded (11.287%) 2024-03-21T12:42:45.770 INFO:journalctl@ceph.osd.1.smithi022.stdout:Mar 21 12:42:45 smithi022 bash[75632]: 0f90e70bbf0a14657e3b081b7429efc76d72f983daf098524e30a26f32010134 2024-03-21T12:42:45.771 INFO:journalctl@ceph.osd.1.smithi022.stdout:Mar 21 12:42:45 smithi022 systemd[1]: Started Ceph osd.1 for 3fc7f0f2-e77e-11ee-95cd-87774f69a715. 2024-03-21T12:42:45.877 INFO:teuthology.orchestra.run.smithi022.stdout: 5 213006 10.88 MB/sec execute 345 sec latency 53.870 ms 2024-03-21T12:42:45.980 INFO:journalctl@ceph.osd.1.smithi022.stdout:Mar 21 12:42:45 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-1[75646]: 2024-03-21T12:42:45.977+0000 7f86cd94d7c0 -1 Falling back to public interface 2024-03-21T12:42:46.739 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:46 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:42:46.740 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:46 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:42:46.740 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:46 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:42:46.740 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:46 smithi022 ceph-mon[67047]: pgmap v104: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 348 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 3.0 KiB/s rd, 2.8 MiB/s wr, 267 op/s; 192/1701 objects degraded (11.287%) 2024-03-21T12:42:46.877 INFO:teuthology.orchestra.run.smithi022.stdout: 5 213388 10.88 MB/sec execute 346 sec latency 39.700 ms 2024-03-21T12:42:47.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:46 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:42:47.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:46 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:42:47.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:46 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:42:47.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:46 smithi189 ceph-mon[53160]: pgmap v104: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 348 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 3.0 KiB/s rd, 2.8 MiB/s wr, 267 op/s; 192/1701 objects degraded (11.287%) 2024-03-21T12:42:47.877 INFO:teuthology.orchestra.run.smithi022.stdout: 5 213797 10.88 MB/sec execute 347 sec latency 42.930 ms 2024-03-21T12:42:48.279 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:48 smithi022 ceph-mon[67047]: Health check update: Degraded data redundancy: 180/1683 objects degraded (10.695%), 24 pgs degraded (PG_DEGRADED) 2024-03-21T12:42:48.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:48 smithi189 ceph-mon[53160]: Health check update: Degraded data redundancy: 180/1683 objects degraded (10.695%), 24 pgs degraded (PG_DEGRADED) 2024-03-21T12:42:48.877 INFO:teuthology.orchestra.run.smithi022.stdout: 5 214231 10.89 MB/sec execute 348 sec latency 51.119 ms 2024-03-21T12:42:49.294 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:49 smithi022 ceph-mon[67047]: pgmap v105: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 349 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.0 MiB/s wr, 504 op/s; 180/1683 objects degraded (10.695%) 2024-03-21T12:42:49.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:49 smithi189 ceph-mon[53160]: pgmap v105: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 349 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.0 MiB/s wr, 504 op/s; 180/1683 objects degraded (10.695%) 2024-03-21T12:42:49.877 INFO:teuthology.orchestra.run.smithi022.stdout: 5 214534 10.88 MB/sec execute 349 sec latency 48.909 ms 2024-03-21T12:42:50.877 INFO:teuthology.orchestra.run.smithi022.stdout: 5 214969 10.88 MB/sec execute 350 sec latency 39.433 ms 2024-03-21T12:42:51.315 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:51 smithi022 ceph-mon[67047]: pgmap v106: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 349 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.4 MiB/s wr, 445 op/s; 180/1683 objects degraded (10.695%) 2024-03-21T12:42:51.315 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:51 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:42:51.315 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:51 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:42:51.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:51 smithi189 ceph-mon[53160]: pgmap v106: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 349 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.4 MiB/s wr, 445 op/s; 180/1683 objects degraded (10.695%) 2024-03-21T12:42:51.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:51 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:42:51.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:51 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:42:51.570 INFO:journalctl@ceph.osd.1.smithi022.stdout:Mar 21 12:42:51 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-1[75646]: 2024-03-21T12:42:51.463+0000 7f86cd94d7c0 -1 osd.1 0 read_superblock omap replica is missing. 2024-03-21T12:42:51.877 INFO:teuthology.orchestra.run.smithi022.stdout: 5 215374 10.89 MB/sec execute 351 sec latency 35.071 ms 2024-03-21T12:42:52.877 INFO:teuthology.orchestra.run.smithi022.stdout: 5 215790 10.89 MB/sec execute 352 sec latency 37.024 ms 2024-03-21T12:42:53.090 INFO:journalctl@ceph.osd.1.smithi022.stdout:Mar 21 12:42:52 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-1[75646]: 2024-03-21T12:42:52.906+0000 7f86cd94d7c0 -1 osd.1 87 log_to_monitors true 2024-03-21T12:42:53.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:53 smithi022 ceph-mon[67047]: pgmap v107: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 352 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 9.1 KiB/s rd, 6.8 MiB/s wr, 674 op/s; 187/1740 objects degraded (10.747%) 2024-03-21T12:42:53.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:53 smithi022 ceph-mon[67047]: from='osd.1 [v2:172.21.15.22:6810/3574241879,v1:172.21.15.22:6811/3574241879]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-03-21T12:42:53.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:53 smithi022 ceph-mon[67047]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-03-21T12:42:53.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:53 smithi189 ceph-mon[53160]: pgmap v107: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 352 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 9.1 KiB/s rd, 6.8 MiB/s wr, 674 op/s; 187/1740 objects degraded (10.747%) 2024-03-21T12:42:53.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:53 smithi189 ceph-mon[53160]: from='osd.1 [v2:172.21.15.22:6810/3574241879,v1:172.21.15.22:6811/3574241879]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-03-21T12:42:53.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:53 smithi189 ceph-mon[53160]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-03-21T12:42:53.854 INFO:journalctl@ceph.osd.1.smithi022.stdout:Mar 21 12:42:53 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-1[75646]: 2024-03-21T12:42:53.605+0000 7f86bad9d700 -1 osd.1 87 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-21T12:42:53.871 INFO:teuthology.orchestra.run.smithi022.stdout:true 2024-03-21T12:42:53.877 INFO:teuthology.orchestra.run.smithi022.stdout: 5 216131 10.88 MB/sec execute 353 sec latency 85.843 ms 2024-03-21T12:42:54.357 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:54 smithi022 ceph-mon[67047]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-03-21T12:42:54.357 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:54 smithi022 ceph-mon[67047]: from='osd.1 [v2:172.21.15.22:6810/3574241879,v1:172.21.15.22:6811/3574241879]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]: dispatch 2024-03-21T12:42:54.357 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:54 smithi022 ceph-mon[67047]: osdmap e90: 8 total, 7 up, 8 in 2024-03-21T12:42:54.357 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:54 smithi022 ceph-mon[67047]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]: dispatch 2024-03-21T12:42:54.357 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:54 smithi022 ceph-mon[67047]: from='osd.1 ' entity='osd.1' 2024-03-21T12:42:54.357 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:54 smithi022 ceph-mon[67047]: Health check update: Degraded data redundancy: 187/1740 objects degraded (10.747%), 24 pgs degraded (PG_DEGRADED) 2024-03-21T12:42:54.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:54 smithi189 ceph-mon[53160]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-03-21T12:42:54.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:54 smithi189 ceph-mon[53160]: from='osd.1 [v2:172.21.15.22:6810/3574241879,v1:172.21.15.22:6811/3574241879]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]: dispatch 2024-03-21T12:42:54.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:54 smithi189 ceph-mon[53160]: osdmap e90: 8 total, 7 up, 8 in 2024-03-21T12:42:54.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:54 smithi189 ceph-mon[53160]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]: dispatch 2024-03-21T12:42:54.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:54 smithi189 ceph-mon[53160]: from='osd.1 ' entity='osd.1' 2024-03-21T12:42:54.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:54 smithi189 ceph-mon[53160]: Health check update: Degraded data redundancy: 187/1740 objects degraded (10.747%), 24 pgs degraded (PG_DEGRADED) 2024-03-21T12:42:54.878 INFO:teuthology.orchestra.run.smithi022.stdout: 5 216434 10.86 MB/sec execute 354 sec latency 227.659 ms 2024-03-21T12:42:54.989 INFO:teuthology.orchestra.run.smithi022.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-21T12:42:54.990 INFO:teuthology.orchestra.run.smithi022.stdout:alertmanager.smithi022 smithi022 *:9093,9094 running (5m) 0s ago 14m 24.1M - 0.23.0 ba2b418f427c 14eb2a3ca318 2024-03-21T12:42:54.990 INFO:teuthology.orchestra.run.smithi022.stdout:crash.smithi022 smithi022 running (94s) 0s ago 14m 6660k - 19.0.0-2267-g960fbbe4 19e12293512f 356b2ddb609a 2024-03-21T12:42:54.990 INFO:teuthology.orchestra.run.smithi022.stdout:crash.smithi189 smithi189 running (88s) 83s ago 13m 6660k - 19.0.0-2267-g960fbbe4 19e12293512f 90ee837f0500 2024-03-21T12:42:54.990 INFO:teuthology.orchestra.run.smithi022.stdout:grafana.smithi022 smithi022 *:3000 error 0s ago 14m - - 2024-03-21T12:42:54.990 INFO:teuthology.orchestra.run.smithi022.stdout:mds.foofs.smithi022.bcnxho smithi022 running (8m) 0s ago 8m 38.1M - 17.2.0 e1d6a67b021e db26b67b8ee7 2024-03-21T12:42:54.990 INFO:teuthology.orchestra.run.smithi022.stdout:mds.foofs.smithi189.lluvko smithi189 running (8m) 83s ago 8m 16.9M - 17.2.0 e1d6a67b021e cf153aff1387 2024-03-21T12:42:54.991 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.smithi022.fektzs smithi022 *:8443,9283,8765 running (4m) 0s ago 15m 503M - 19.0.0-2267-g960fbbe4 19e12293512f d57cebd306b3 2024-03-21T12:42:54.991 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.smithi189.cnxzvu smithi189 *:8443,9283,8765 running (3m) 83s ago 13m 450M - 19.0.0-2267-g960fbbe4 19e12293512f 41663955c56f 2024-03-21T12:42:54.991 INFO:teuthology.orchestra.run.smithi022.stdout:mon.smithi022 smithi022 running (3m) 0s ago 15m 48.2M 2048M 19.0.0-2267-g960fbbe4 19e12293512f c620c9071b2c 2024-03-21T12:42:54.991 INFO:teuthology.orchestra.run.smithi022.stdout:mon.smithi189 smithi189 running (2m) 83s ago 13m 35.8M 2048M 19.0.0-2267-g960fbbe4 19e12293512f b353af3d8f3b 2024-03-21T12:42:54.991 INFO:teuthology.orchestra.run.smithi022.stdout:nfs.foo.0.0.smithi022.ynhbbq smithi022 *:2049 running (8m) 0s ago 8m 276M - 4.0 e1d6a67b021e 33e48e64c60b 2024-03-21T12:42:54.991 INFO:teuthology.orchestra.run.smithi022.stdout:nfs.foo.1.0.smithi189.yzsqml smithi189 *:2049 running (8m) 83s ago 8m 85.4M - 4.0 e1d6a67b021e 48fab2cbdde2 2024-03-21T12:42:54.991 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.smithi022 smithi022 *:9100 running (5m) 0s ago 13m 21.0M - 1.3.1 1dbe0e931976 33287f803f4f 2024-03-21T12:42:54.991 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.smithi189 smithi189 *:9100 running (4m) 83s ago 13m 17.5M - 1.3.1 1dbe0e931976 b51d872e438a 2024-03-21T12:42:54.991 INFO:teuthology.orchestra.run.smithi022.stdout:osd.0 smithi022 running (56s) 0s ago 12m 236M 1937M 19.0.0-2267-g960fbbe4 19e12293512f 23a85645eeea 2024-03-21T12:42:54.991 INFO:teuthology.orchestra.run.smithi022.stdout:osd.1 smithi022 running (10s) 0s ago 12m 12.2M 1937M 19.0.0-2267-g960fbbe4 19e12293512f 0f90e70bbf0a 2024-03-21T12:42:54.991 INFO:teuthology.orchestra.run.smithi022.stdout:osd.2 smithi022 running (11m) 0s ago 11m 513M 1937M 17.2.0 e1d6a67b021e fbec9bef8433 2024-03-21T12:42:54.991 INFO:teuthology.orchestra.run.smithi022.stdout:osd.3 smithi022 running (11m) 0s ago 11m 386M 1937M 17.2.0 e1d6a67b021e b1462ffc1b9e 2024-03-21T12:42:54.991 INFO:teuthology.orchestra.run.smithi022.stdout:osd.4 smithi189 running (11m) 83s ago 11m 560M 2704M 17.2.0 e1d6a67b021e 1bedc4eb02ed 2024-03-21T12:42:54.991 INFO:teuthology.orchestra.run.smithi022.stdout:osd.5 smithi189 running (10m) 83s ago 10m 407M 2704M 17.2.0 e1d6a67b021e 25fbab9786a5 2024-03-21T12:42:54.991 INFO:teuthology.orchestra.run.smithi022.stdout:osd.6 smithi189 running (10m) 83s ago 10m 578M 2704M 17.2.0 e1d6a67b021e 0c192fcfd65b 2024-03-21T12:42:54.991 INFO:teuthology.orchestra.run.smithi022.stdout:osd.7 smithi189 running (9m) 83s ago 9m 528M 2704M 17.2.0 e1d6a67b021e 3d0723e1f863 2024-03-21T12:42:54.992 INFO:teuthology.orchestra.run.smithi022.stdout:prometheus.smithi022 smithi022 *:9095 running (3m) 0s ago 13m 73.2M - 2.33.4 514e6a882f6e 8171bbe41d2c 2024-03-21T12:42:55.408 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:55 smithi022 ceph-mon[67047]: from='client.34254 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:42:55.408 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:55 smithi022 ceph-mon[67047]: pgmap v109: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 352 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.8 MiB/s wr, 586 op/s; 187/1740 objects degraded (10.747%) 2024-03-21T12:42:55.408 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:55 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:42:55.408 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:55 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:42:55.408 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:55 smithi022 ceph-mon[67047]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-21T12:42:55.408 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:55 smithi022 ceph-mon[67047]: osd.1 [v2:172.21.15.22:6810/3574241879,v1:172.21.15.22:6811/3574241879] boot 2024-03-21T12:42:55.408 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:55 smithi022 ceph-mon[67047]: osdmap e91: 8 total, 8 up, 8 in 2024-03-21T12:42:55.408 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:55 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-21T12:42:55.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:55 smithi189 ceph-mon[53160]: from='client.34254 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:42:55.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:55 smithi189 ceph-mon[53160]: pgmap v109: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 352 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.8 MiB/s wr, 586 op/s; 187/1740 objects degraded (10.747%) 2024-03-21T12:42:55.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:55 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:42:55.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:55 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:42:55.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:55 smithi189 ceph-mon[53160]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-21T12:42:55.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:55 smithi189 ceph-mon[53160]: osd.1 [v2:172.21.15.22:6810/3574241879,v1:172.21.15.22:6811/3574241879] boot 2024-03-21T12:42:55.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:55 smithi189 ceph-mon[53160]: osdmap e91: 8 total, 8 up, 8 in 2024-03-21T12:42:55.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:55 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-21T12:42:55.580 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-03-21T12:42:55.580 INFO:teuthology.orchestra.run.smithi022.stdout: "mon": { 2024-03-21T12:42:55.580 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 2 2024-03-21T12:42:55.580 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:42:55.580 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr": { 2024-03-21T12:42:55.580 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 2 2024-03-21T12:42:55.580 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:42:55.580 INFO:teuthology.orchestra.run.smithi022.stdout: "osd": { 2024-03-21T12:42:55.580 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-03-21T12:42:55.580 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 2 2024-03-21T12:42:55.580 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:42:55.580 INFO:teuthology.orchestra.run.smithi022.stdout: "mds": { 2024-03-21T12:42:55.580 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-21T12:42:55.580 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:42:55.580 INFO:teuthology.orchestra.run.smithi022.stdout: "overall": { 2024-03-21T12:42:55.580 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8, 2024-03-21T12:42:55.581 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 6 2024-03-21T12:42:55.581 INFO:teuthology.orchestra.run.smithi022.stdout: } 2024-03-21T12:42:55.581 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-03-21T12:42:55.878 INFO:teuthology.orchestra.run.smithi022.stdout: 5 216572 10.84 MB/sec execute 355 sec latency 1227.733 ms 2024-03-21T12:42:56.113 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-03-21T12:42:56.113 INFO:teuthology.orchestra.run.smithi022.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93", 2024-03-21T12:42:56.113 INFO:teuthology.orchestra.run.smithi022.stdout: "in_progress": true, 2024-03-21T12:42:56.113 INFO:teuthology.orchestra.run.smithi022.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-21T12:42:56.113 INFO:teuthology.orchestra.run.smithi022.stdout: "services_complete": [ 2024-03-21T12:42:56.113 INFO:teuthology.orchestra.run.smithi022.stdout: "mon", 2024-03-21T12:42:56.113 INFO:teuthology.orchestra.run.smithi022.stdout: "crash", 2024-03-21T12:42:56.114 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr" 2024-03-21T12:42:56.114 INFO:teuthology.orchestra.run.smithi022.stdout: ], 2024-03-21T12:42:56.114 INFO:teuthology.orchestra.run.smithi022.stdout: "progress": "8/23 daemons upgraded", 2024-03-21T12:42:56.114 INFO:teuthology.orchestra.run.smithi022.stdout: "message": "Currently upgrading osd daemons", 2024-03-21T12:42:56.114 INFO:teuthology.orchestra.run.smithi022.stdout: "is_paused": false 2024-03-21T12:42:56.114 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-03-21T12:42:56.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:56 smithi189 ceph-mon[53160]: from='client.34258 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:42:56.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:56 smithi189 ceph-mon[53160]: from='client.34262 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:42:56.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:56 smithi189 ceph-mon[53160]: from='client.? 172.21.15.22:0/3579929339' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:42:56.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:56 smithi189 ceph-mon[53160]: osdmap e92: 8 total, 8 up, 8 in 2024-03-21T12:42:56.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:56 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:42:56.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:56 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:42:56.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:56 smithi022 ceph-mon[67047]: from='client.34258 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:42:56.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:56 smithi022 ceph-mon[67047]: from='client.34262 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:42:56.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:56 smithi022 ceph-mon[67047]: from='client.? 172.21.15.22:0/3579929339' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:42:56.595 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:56 smithi022 ceph-mon[67047]: osdmap e92: 8 total, 8 up, 8 in 2024-03-21T12:42:56.595 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:56 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:42:56.595 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:56 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:42:56.634 INFO:teuthology.orchestra.run.smithi022.stdout:HEALTH_WARN 1 failed cephadm daemon(s); Degraded data redundancy: 187/1740 objects degraded (10.747%), 24 pgs degraded 2024-03-21T12:42:56.634 INFO:teuthology.orchestra.run.smithi022.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-21T12:42:56.634 INFO:teuthology.orchestra.run.smithi022.stdout: daemon grafana.smithi022 on smithi022 is in error state 2024-03-21T12:42:56.634 INFO:teuthology.orchestra.run.smithi022.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 187/1740 objects degraded (10.747%), 24 pgs degraded 2024-03-21T12:42:56.635 INFO:teuthology.orchestra.run.smithi022.stdout: pg 2.0 is active+undersized+degraded, acting [7,0] 2024-03-21T12:42:56.635 INFO:teuthology.orchestra.run.smithi022.stdout: pg 2.2 is active+undersized+degraded, acting [5,6] 2024-03-21T12:42:56.635 INFO:teuthology.orchestra.run.smithi022.stdout: pg 2.4 is active+undersized+degraded, acting [0,7] 2024-03-21T12:42:56.635 INFO:teuthology.orchestra.run.smithi022.stdout: pg 2.6 is active+undersized+degraded, acting [6,4] 2024-03-21T12:42:56.635 INFO:teuthology.orchestra.run.smithi022.stdout: pg 2.8 is active+undersized+degraded, acting [2,0] 2024-03-21T12:42:56.635 INFO:teuthology.orchestra.run.smithi022.stdout: pg 2.9 is active+undersized+degraded, acting [7,3] 2024-03-21T12:42:56.635 INFO:teuthology.orchestra.run.smithi022.stdout: pg 2.a is active+undersized+degraded, acting [3,7] 2024-03-21T12:42:56.635 INFO:teuthology.orchestra.run.smithi022.stdout: pg 2.d is active+undersized+degraded, acting [4,3] 2024-03-21T12:42:56.635 INFO:teuthology.orchestra.run.smithi022.stdout: pg 2.10 is active+undersized+degraded, acting [2,0] 2024-03-21T12:42:56.635 INFO:teuthology.orchestra.run.smithi022.stdout: pg 2.11 is active+undersized+degraded, acting [6,4] 2024-03-21T12:42:56.635 INFO:teuthology.orchestra.run.smithi022.stdout: pg 2.15 is active+undersized+degraded, acting [5,0] 2024-03-21T12:42:56.635 INFO:teuthology.orchestra.run.smithi022.stdout: pg 3.0 is active+undersized+degraded, acting [2,6] 2024-03-21T12:42:56.635 INFO:teuthology.orchestra.run.smithi022.stdout: pg 3.4 is active+undersized+degraded, acting [2,5] 2024-03-21T12:42:56.635 INFO:teuthology.orchestra.run.smithi022.stdout: pg 3.6 is active+undersized+degraded, acting [0,4] 2024-03-21T12:42:56.635 INFO:teuthology.orchestra.run.smithi022.stdout: pg 3.8 is active+undersized+degraded, acting [3,7] 2024-03-21T12:42:56.635 INFO:teuthology.orchestra.run.smithi022.stdout: pg 3.a is active+undersized+degraded, acting [6,4] 2024-03-21T12:42:56.636 INFO:teuthology.orchestra.run.smithi022.stdout: pg 3.e is active+undersized+degraded, acting [7,4] 2024-03-21T12:42:56.636 INFO:teuthology.orchestra.run.smithi022.stdout: pg 3.16 is active+undersized+degraded, acting [5,7] 2024-03-21T12:42:56.636 INFO:teuthology.orchestra.run.smithi022.stdout: pg 3.18 is active+undersized+degraded, acting [3,0] 2024-03-21T12:42:56.636 INFO:teuthology.orchestra.run.smithi022.stdout: pg 3.19 is active+undersized+degraded, acting [3,4] 2024-03-21T12:42:56.636 INFO:teuthology.orchestra.run.smithi022.stdout: pg 3.1a is active+undersized+degraded, acting [4,2] 2024-03-21T12:42:56.636 INFO:teuthology.orchestra.run.smithi022.stdout: pg 3.1c is active+undersized+degraded, acting [5,4] 2024-03-21T12:42:56.636 INFO:teuthology.orchestra.run.smithi022.stdout: pg 4.a is active+undersized+degraded, acting [6,7] 2024-03-21T12:42:56.636 INFO:teuthology.orchestra.run.smithi022.stdout: pg 4.14 is active+undersized+degraded, acting [3,7] 2024-03-21T12:42:56.878 INFO:teuthology.orchestra.run.smithi022.stdout: 5 216710 10.82 MB/sec execute 356 sec latency 1996.086 ms 2024-03-21T12:42:57.392 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:57 smithi022 ceph-mon[67047]: pgmap v112: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 352 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.2 MiB/s wr, 507 op/s; 187/1740 objects degraded (10.747%) 2024-03-21T12:42:57.393 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:57 smithi022 ceph-mon[67047]: from='client.44187 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:42:57.393 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:57 smithi022 ceph-mon[67047]: osdmap e93: 8 total, 8 up, 8 in 2024-03-21T12:42:57.393 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:57 smithi022 ceph-mon[67047]: from='client.? 172.21.15.22:0/4083573146' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-21T12:42:57.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:57 smithi189 ceph-mon[53160]: pgmap v112: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 352 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.2 MiB/s wr, 507 op/s; 187/1740 objects degraded (10.747%) 2024-03-21T12:42:57.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:57 smithi189 ceph-mon[53160]: from='client.44187 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:42:57.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:57 smithi189 ceph-mon[53160]: osdmap e93: 8 total, 8 up, 8 in 2024-03-21T12:42:57.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:57 smithi189 ceph-mon[53160]: from='client.? 172.21.15.22:0/4083573146' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-21T12:42:57.878 INFO:teuthology.orchestra.run.smithi022.stdout: 5 217139 10.82 MB/sec execute 357 sec latency 43.898 ms 2024-03-21T12:42:58.649 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:58 smithi022 ceph-mon[67047]: osdmap e94: 8 total, 8 up, 8 in 2024-03-21T12:42:58.650 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:58 smithi022 ceph-mon[67047]: pgmap v115: 97 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+degraded, 1 activating+undersized+degraded+remapped, 4 active+undersized, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 7 active+undersized+degraded, 76 active+clean; 345 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 4.6 MiB/s wr, 491 op/s; 199/1509 objects degraded (13.188%); 1024 KiB/s, 5 objects/s recovering 2024-03-21T12:42:58.878 INFO:teuthology.orchestra.run.smithi022.stdout: 5 217635 10.83 MB/sec execute 358 sec latency 121.334 ms 2024-03-21T12:42:59.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:58 smithi189 ceph-mon[53160]: osdmap e94: 8 total, 8 up, 8 in 2024-03-21T12:42:59.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:58 smithi189 ceph-mon[53160]: pgmap v115: 97 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+degraded, 1 activating+undersized+degraded+remapped, 4 active+undersized, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 7 active+undersized+degraded, 76 active+clean; 345 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 4.6 MiB/s wr, 491 op/s; 199/1509 objects degraded (13.188%); 1024 KiB/s, 5 objects/s recovering 2024-03-21T12:42:59.728 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:59 smithi022 ceph-mon[67047]: osdmap e95: 8 total, 8 up, 8 in 2024-03-21T12:42:59.728 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:59 smithi022 ceph-mon[67047]: Health check update: Degraded data redundancy: 199/1509 objects degraded (13.188%), 16 pgs degraded (PG_DEGRADED) 2024-03-21T12:42:59.728 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:42:59 smithi022 ceph-mon[67047]: osdmap e96: 8 total, 8 up, 8 in 2024-03-21T12:42:59.878 INFO:teuthology.orchestra.run.smithi022.stdout: 5 217963 10.83 MB/sec execute 359 sec latency 454.534 ms 2024-03-21T12:43:00.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:59 smithi189 ceph-mon[53160]: osdmap e95: 8 total, 8 up, 8 in 2024-03-21T12:43:00.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:59 smithi189 ceph-mon[53160]: Health check update: Degraded data redundancy: 199/1509 objects degraded (13.188%), 16 pgs degraded (PG_DEGRADED) 2024-03-21T12:43:00.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:42:59 smithi189 ceph-mon[53160]: osdmap e96: 8 total, 8 up, 8 in 2024-03-21T12:43:00.857 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:00 smithi022 ceph-mon[67047]: osdmap e97: 8 total, 8 up, 8 in 2024-03-21T12:43:00.857 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:00 smithi022 ceph-mon[67047]: pgmap v119: 97 pgs: 1 peering, 1 active+recovery_wait+degraded, 1 active+recovering+degraded, 1 activating+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 345 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 5.4 MiB/s wr, 584 op/s; 152/1506 objects degraded (10.093%); 25 B/s, 79 keys/s, 16 objects/s recovering 2024-03-21T12:43:00.878 INFO:teuthology.orchestra.run.smithi022.stdout: 5 218462 10.85 MB/sec execute 360 sec latency 28.537 ms 2024-03-21T12:43:01.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:00 smithi189 ceph-mon[53160]: osdmap e97: 8 total, 8 up, 8 in 2024-03-21T12:43:01.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:00 smithi189 ceph-mon[53160]: pgmap v119: 97 pgs: 1 peering, 1 active+recovery_wait+degraded, 1 active+recovering+degraded, 1 activating+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 345 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 5.4 MiB/s wr, 584 op/s; 152/1506 objects degraded (10.093%); 25 B/s, 79 keys/s, 16 objects/s recovering 2024-03-21T12:43:01.878 INFO:teuthology.orchestra.run.smithi022.stdout: 5 219042 10.86 MB/sec execute 361 sec latency 24.522 ms 2024-03-21T12:43:01.986 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:01 smithi022 ceph-mon[67047]: osdmap e98: 8 total, 8 up, 8 in 2024-03-21T12:43:02.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:01 smithi189 ceph-mon[53160]: osdmap e98: 8 total, 8 up, 8 in 2024-03-21T12:43:02.878 INFO:teuthology.orchestra.run.smithi022.stdout: 5 219367 10.86 MB/sec execute 362 sec latency 37.243 ms 2024-03-21T12:43:03.001 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:02 smithi022 ceph-mon[67047]: osdmap e99: 8 total, 8 up, 8 in 2024-03-21T12:43:03.002 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:02 smithi022 ceph-mon[67047]: pgmap v122: 97 pgs: 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 354 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 9.7 MiB/s wr, 868 op/s; 25/1731 objects degraded (1.444%); 1.3 MiB/s, 39 objects/s recovering 2024-03-21T12:43:03.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:02 smithi189 ceph-mon[53160]: osdmap e99: 8 total, 8 up, 8 in 2024-03-21T12:43:03.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:02 smithi189 ceph-mon[53160]: pgmap v122: 97 pgs: 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 354 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 9.7 MiB/s wr, 868 op/s; 25/1731 objects degraded (1.444%); 1.3 MiB/s, 39 objects/s recovering 2024-03-21T12:43:03.878 INFO:teuthology.orchestra.run.smithi022.stdout: 5 219692 10.84 MB/sec execute 363 sec latency 65.796 ms 2024-03-21T12:43:04.026 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:03 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:43:04.026 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:03 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:43:04.026 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:03 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:43:04.026 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:03 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:43:04.026 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:03 smithi022 ceph-mon[67047]: pgmap v123: 97 pgs: 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 354 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 8.3 MiB/s wr, 742 op/s; 25/1731 objects degraded (1.444%); 1.1 MiB/s, 33 objects/s recovering 2024-03-21T12:43:04.026 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:03 smithi022 ceph-mon[67047]: pgmap v124: 97 pgs: 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 354 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 11 MiB/s wr, 985 op/s; 25/1731 objects degraded (1.444%); 1.5 MiB/s, 45 objects/s recovering 2024-03-21T12:43:04.026 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:03 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:43:04.026 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:03 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:43:04.026 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:03 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:43:04.027 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:03 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:43:04.027 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:03 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:43:04.027 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:03 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:43:04.027 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:03 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-21T12:43:04.027 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:03 smithi022 ceph-mon[67047]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-21T12:43:04.027 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:03 smithi022 ceph-mon[67047]: Upgrade: osd.2 is safe to restart 2024-03-21T12:43:04.027 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:03 smithi022 ceph-mon[67047]: Health check update: Degraded data redundancy: 25/1731 objects degraded (1.444%), 1 pg degraded (PG_DEGRADED) 2024-03-21T12:43:04.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:03 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:43:04.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:03 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:43:04.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:03 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:43:04.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:03 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:43:04.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:03 smithi189 ceph-mon[53160]: pgmap v123: 97 pgs: 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 354 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 8.3 MiB/s wr, 742 op/s; 25/1731 objects degraded (1.444%); 1.1 MiB/s, 33 objects/s recovering 2024-03-21T12:43:04.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:03 smithi189 ceph-mon[53160]: pgmap v124: 97 pgs: 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 354 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 11 MiB/s wr, 985 op/s; 25/1731 objects degraded (1.444%); 1.5 MiB/s, 45 objects/s recovering 2024-03-21T12:43:04.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:03 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:43:04.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:03 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:43:04.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:03 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:43:04.287 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:03 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:43:04.287 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:03 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:43:04.287 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:03 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:43:04.287 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:03 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-21T12:43:04.287 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:03 smithi189 ceph-mon[53160]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-21T12:43:04.287 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:03 smithi189 ceph-mon[53160]: Upgrade: osd.2 is safe to restart 2024-03-21T12:43:04.287 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:03 smithi189 ceph-mon[53160]: Health check update: Degraded data redundancy: 25/1731 objects degraded (1.444%), 1 pg degraded (PG_DEGRADED) 2024-03-21T12:43:04.878 INFO:teuthology.orchestra.run.smithi022.stdout: 5 220044 10.83 MB/sec execute 364 sec latency 43.254 ms 2024-03-21T12:43:05.038 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:04 smithi022 ceph-mon[67047]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 25/1731 objects degraded (1.444%), 1 pg degraded) 2024-03-21T12:43:05.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:04 smithi189 ceph-mon[53160]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 25/1731 objects degraded (1.444%), 1 pg degraded) 2024-03-21T12:43:05.878 INFO:teuthology.orchestra.run.smithi022.stdout: 5 220348 10.81 MB/sec execute 365 sec latency 42.680 ms 2024-03-21T12:43:06.084 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:05 smithi022 ceph-mon[67047]: pgmap v125: 97 pgs: 97 active+clean; 354 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 7.3 MiB/s wr, 659 op/s; 1.8 MiB/s, 34 objects/s recovering 2024-03-21T12:43:06.084 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:05 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:43:06.084 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:05 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:43:06.084 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:05 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:43:06.084 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:05 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-03-21T12:43:06.084 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:05 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:43:06.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:05 smithi189 ceph-mon[53160]: pgmap v125: 97 pgs: 97 active+clean; 354 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 7.3 MiB/s wr, 659 op/s; 1.8 MiB/s, 34 objects/s recovering 2024-03-21T12:43:06.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:05 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:43:06.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:05 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:43:06.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:05 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:43:06.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:05 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-03-21T12:43:06.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:05 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:43:06.879 INFO:teuthology.orchestra.run.smithi022.stdout: 5 220652 10.81 MB/sec execute 366 sec latency 35.052 ms 2024-03-21T12:43:07.108 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:06 smithi022 ceph-mon[67047]: Upgrade: Updating osd.2 2024-03-21T12:43:07.109 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:06 smithi022 ceph-mon[67047]: Deploying daemon osd.2 on smithi022 2024-03-21T12:43:07.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:06 smithi189 ceph-mon[53160]: Upgrade: Updating osd.2 2024-03-21T12:43:07.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:06 smithi189 ceph-mon[53160]: Deploying daemon osd.2 on smithi022 2024-03-21T12:43:07.878 INFO:teuthology.orchestra.run.smithi022.stdout: 5 221003 10.81 MB/sec execute 367 sec latency 46.991 ms 2024-03-21T12:43:08.139 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:07 smithi022 ceph-mon[67047]: pgmap v126: 97 pgs: 97 active+clean; 347 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 14 KiB/s rd, 8.8 MiB/s wr, 914 op/s; 1.4 MiB/s, 27 objects/s recovering 2024-03-21T12:43:08.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:07 smithi189 ceph-mon[53160]: pgmap v126: 97 pgs: 97 active+clean; 347 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 14 KiB/s rd, 8.8 MiB/s wr, 914 op/s; 1.4 MiB/s, 27 objects/s recovering 2024-03-21T12:43:08.879 INFO:teuthology.orchestra.run.smithi022.stdout: 5 221410 10.81 MB/sec execute 368 sec latency 47.500 ms 2024-03-21T12:43:09.879 INFO:teuthology.orchestra.run.smithi022.stdout: 5 221709 10.81 MB/sec execute 369 sec latency 36.224 ms 2024-03-21T12:43:10.175 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:09 smithi022 ceph-mon[67047]: pgmap v127: 97 pgs: 97 active+clean; 346 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 4.2 MiB/s wr, 416 op/s; 655 KiB/s, 6 objects/s recovering 2024-03-21T12:43:10.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:09 smithi189 ceph-mon[53160]: pgmap v127: 97 pgs: 97 active+clean; 346 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 4.2 MiB/s wr, 416 op/s; 655 KiB/s, 6 objects/s recovering 2024-03-21T12:43:10.879 INFO:teuthology.orchestra.run.smithi022.stdout: 5 222056 10.82 MB/sec execute 370 sec latency 45.645 ms 2024-03-21T12:43:10.889 INFO:journalctl@ceph.osd.2.smithi022.stdout:Mar 21 12:43:10 smithi022 systemd[1]: Stopping Ceph osd.2 for 3fc7f0f2-e77e-11ee-95cd-87774f69a715... 2024-03-21T12:43:11.194 INFO:journalctl@ceph.osd.2.smithi022.stdout:Mar 21 12:43:11 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-2[41999]: 2024-03-21T12:43:11.176+0000 7f943d2f3700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-21T12:43:11.194 INFO:journalctl@ceph.osd.2.smithi022.stdout:Mar 21 12:43:11 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-2[41999]: 2024-03-21T12:43:11.176+0000 7f943d2f3700 -1 osd.2 99 *** Got signal Terminated *** 2024-03-21T12:43:11.194 INFO:journalctl@ceph.osd.2.smithi022.stdout:Mar 21 12:43:11 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-2[41999]: 2024-03-21T12:43:11.176+0000 7f943d2f3700 -1 osd.2 99 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-21T12:43:11.879 INFO:teuthology.orchestra.run.smithi022.stdout: 5 222434 10.82 MB/sec execute 371 sec latency 62.482 ms 2024-03-21T12:43:11.964 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:11 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:43:11.965 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:11 smithi022 ceph-mon[67047]: pgmap v128: 97 pgs: 97 active+clean; 346 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 2.8 MiB/s wr, 318 op/s; 470 KiB/s, 2 objects/s recovering 2024-03-21T12:43:11.965 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:11 smithi022 ceph-mon[67047]: osd.2 marked itself down and dead 2024-03-21T12:43:12.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:11 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:43:12.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:11 smithi189 ceph-mon[53160]: pgmap v128: 97 pgs: 97 active+clean; 346 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 2.8 MiB/s wr, 318 op/s; 470 KiB/s, 2 objects/s recovering 2024-03-21T12:43:12.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:11 smithi189 ceph-mon[53160]: osd.2 marked itself down and dead 2024-03-21T12:43:12.468 INFO:journalctl@ceph.osd.2.smithi022.stdout:Mar 21 12:43:12 smithi022 podman[77694]: 2024-03-21 12:43:12.418165206 +0000 UTC m=+1.475045446 container died fbec9bef8433ade720143dc514fb6e36561f5bf91a12261286be1edd531d29ad (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-2, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, release=754, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, name=centos-stream, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-03-21T12:43:12.879 INFO:teuthology.orchestra.run.smithi022.stdout: 5 222600 10.79 MB/sec execute 372 sec latency 949.983 ms 2024-03-21T12:43:12.989 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:12 smithi022 ceph-mon[67047]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-21T12:43:12.990 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:12 smithi022 ceph-mon[67047]: osdmap e100: 8 total, 7 up, 8 in 2024-03-21T12:43:13.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:12 smithi189 ceph-mon[53160]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-21T12:43:13.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:12 smithi189 ceph-mon[53160]: osdmap e100: 8 total, 7 up, 8 in 2024-03-21T12:43:13.254 INFO:journalctl@ceph.osd.2.smithi022.stdout:Mar 21 12:43:13 smithi022 podman[77694]: 2024-03-21 12:43:13.167947975 +0000 UTC m=+2.224828202 container cleanup fbec9bef8433ade720143dc514fb6e36561f5bf91a12261286be1edd531d29ad (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-2, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., version=8, CEPH_POINT_RELEASE=-17.2.0, release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, GIT_CLEAN=True, architecture=x86_64, RELEASE=HEAD, ceph=True, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD) 2024-03-21T12:43:13.255 INFO:journalctl@ceph.osd.2.smithi022.stdout:Mar 21 12:43:13 smithi022 bash[77694]: ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-2 2024-03-21T12:43:13.756 INFO:journalctl@ceph.osd.2.smithi022.stdout:Mar 21 12:43:13 smithi022 podman[77714]: 2024-03-21 12:43:13.505292983 +0000 UTC m=+1.088224342 container remove fbec9bef8433ade720143dc514fb6e36561f5bf91a12261286be1edd531d29ad (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-2, io.buildah.version=1.19.8, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, ceph=True, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, release=754, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8) 2024-03-21T12:43:13.757 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:13 smithi022 ceph-mon[67047]: osdmap e101: 8 total, 7 up, 8 in 2024-03-21T12:43:13.757 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:13 smithi022 ceph-mon[67047]: pgmap v131: 97 pgs: 3 peering, 6 stale+active+clean, 88 active+clean; 353 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 9.9 KiB/s rd, 6.3 MiB/s wr, 628 op/s 2024-03-21T12:43:13.879 INFO:teuthology.orchestra.run.smithi022.stdout: 5 222793 10.78 MB/sec execute 373 sec latency 71.925 ms 2024-03-21T12:43:14.010 INFO:journalctl@ceph.osd.2.smithi022.stdout:Mar 21 12:43:13 smithi022 podman[77806]: 2024-03-21 12:43:13.821706111 +0000 UTC m=+0.020485592 image pull 19e12293512ff7978c8adb5f72a829060dd47f07bfd6903b6b3178d2ebc29e4e quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93 2024-03-21T12:43:14.011 INFO:journalctl@ceph.osd.2.smithi022.stdout:Mar 21 12:43:13 smithi022 podman[77806]: 2024-03-21 12:43:13.939736766 +0000 UTC m=+0.138516232 container create a5b4be5c8f47ca3a6c3c40a555d9df89222a21efa690018d08e73720950f361e (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-2-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.build-date=20240319, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_CLEAN=True) 2024-03-21T12:43:14.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:13 smithi189 ceph-mon[53160]: osdmap e101: 8 total, 7 up, 8 in 2024-03-21T12:43:14.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:13 smithi189 ceph-mon[53160]: pgmap v131: 97 pgs: 3 peering, 6 stale+active+clean, 88 active+clean; 353 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 9.9 KiB/s rd, 6.3 MiB/s wr, 628 op/s 2024-03-21T12:43:14.258 INFO:journalctl@ceph.osd.2.smithi022.stdout:Mar 21 12:43:14 smithi022 podman[77806]: 2024-03-21 12:43:14.242888093 +0000 UTC m=+0.441667566 container init a5b4be5c8f47ca3a6c3c40a555d9df89222a21efa690018d08e73720950f361e (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-2-deactivate, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240319, ceph=True, org.label-schema.schema-version=1.0) 2024-03-21T12:43:14.259 INFO:journalctl@ceph.osd.2.smithi022.stdout:Mar 21 12:43:14 smithi022 podman[77806]: 2024-03-21 12:43:14.251841346 +0000 UTC m=+0.450620810 container start a5b4be5c8f47ca3a6c3c40a555d9df89222a21efa690018d08e73720950f361e (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-2-deactivate, org.label-schema.vendor=CentOS, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, org.label-schema.build-date=20240319, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True) 2024-03-21T12:43:14.509 INFO:journalctl@ceph.osd.2.smithi022.stdout:Mar 21 12:43:14 smithi022 podman[77806]: 2024-03-21 12:43:14.393725415 +0000 UTC m=+0.592504894 container attach a5b4be5c8f47ca3a6c3c40a555d9df89222a21efa690018d08e73720950f361e (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-2-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240319, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, ceph=True, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-21T12:43:14.771 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:14 smithi022 ceph-mon[67047]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-03-21T12:43:14.771 INFO:journalctl@ceph.osd.2.smithi022.stdout:Mar 21 12:43:14 smithi022 podman[77806]: 2024-03-21 12:43:14.664954911 +0000 UTC m=+0.863734378 container died a5b4be5c8f47ca3a6c3c40a555d9df89222a21efa690018d08e73720950f361e (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-2-deactivate, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.build-date=20240319, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_CLEAN=True) 2024-03-21T12:43:14.879 INFO:teuthology.orchestra.run.smithi022.stdout: 5 223012 10.77 MB/sec execute 374 sec latency 61.372 ms 2024-03-21T12:43:15.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:14 smithi189 ceph-mon[53160]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-03-21T12:43:15.779 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:15 smithi022 ceph-mon[67047]: pgmap v132: 97 pgs: 3 active+undersized, 3 active+undersized+degraded, 3 peering, 4 stale+active+clean, 84 active+clean; 354 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 2.7 KiB/s rd, 4.1 MiB/s wr, 331 op/s; 33/1716 objects degraded (1.923%) 2024-03-21T12:43:15.879 INFO:teuthology.orchestra.run.smithi022.stdout: 5 223240 10.75 MB/sec execute 375 sec latency 93.865 ms 2024-03-21T12:43:16.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:15 smithi189 ceph-mon[53160]: pgmap v132: 97 pgs: 3 active+undersized, 3 active+undersized+degraded, 3 peering, 4 stale+active+clean, 84 active+clean; 354 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 2.7 KiB/s rd, 4.1 MiB/s wr, 331 op/s; 33/1716 objects degraded (1.923%) 2024-03-21T12:43:16.049 INFO:journalctl@ceph.osd.2.smithi022.stdout:Mar 21 12:43:15 smithi022 podman[77831]: 2024-03-21 12:43:15.811661288 +0000 UTC m=+1.140648978 container remove a5b4be5c8f47ca3a6c3c40a555d9df89222a21efa690018d08e73720950f361e (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-2-deactivate, maintainer=Guillaume Abrioux , GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240319, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-03-21T12:43:16.050 INFO:journalctl@ceph.osd.2.smithi022.stdout:Mar 21 12:43:15 smithi022 systemd[1]: ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715@osd.2.service: Deactivated successfully. 2024-03-21T12:43:16.050 INFO:journalctl@ceph.osd.2.smithi022.stdout:Mar 21 12:43:15 smithi022 systemd[1]: Stopped Ceph osd.2 for 3fc7f0f2-e77e-11ee-95cd-87774f69a715. 2024-03-21T12:43:16.050 INFO:journalctl@ceph.osd.2.smithi022.stdout:Mar 21 12:43:15 smithi022 systemd[1]: ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715@osd.2.service: Consumed 2min 48.092s CPU time. 2024-03-21T12:43:16.306 INFO:journalctl@ceph.osd.2.smithi022.stdout:Mar 21 12:43:16 smithi022 systemd[1]: Starting Ceph osd.2 for 3fc7f0f2-e77e-11ee-95cd-87774f69a715... 2024-03-21T12:43:16.573 INFO:journalctl@ceph.osd.2.smithi022.stdout:Mar 21 12:43:16 smithi022 podman[77934]: 2024-03-21 12:43:16.284213041 +0000 UTC m=+0.030057810 image pull 19e12293512ff7978c8adb5f72a829060dd47f07bfd6903b6b3178d2ebc29e4e quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93 2024-03-21T12:43:16.574 INFO:journalctl@ceph.osd.2.smithi022.stdout:Mar 21 12:43:16 smithi022 podman[77934]: 2024-03-21 12:43:16.443128292 +0000 UTC m=+0.188973045 container create 1e6261e53b5cf528827ce5035a0499cdb1ddfe121796f70e085ff4701b0bfc8a (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-2-activate, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, ceph=True, org.label-schema.build-date=20240319, GIT_BRANCH=HEAD, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-21T12:43:16.824 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:16 smithi022 ceph-mon[67047]: Health check failed: Degraded data redundancy: 33/1716 objects degraded (1.923%), 3 pgs degraded (PG_DEGRADED) 2024-03-21T12:43:16.879 INFO:teuthology.orchestra.run.smithi022.stdout: 5 223643 10.74 MB/sec execute 376 sec latency 57.922 ms 2024-03-21T12:43:17.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:16 smithi189 ceph-mon[53160]: Health check failed: Degraded data redundancy: 33/1716 objects degraded (1.923%), 3 pgs degraded (PG_DEGRADED) 2024-03-21T12:43:17.078 INFO:journalctl@ceph.osd.2.smithi022.stdout:Mar 21 12:43:16 smithi022 podman[77934]: 2024-03-21 12:43:16.843410284 +0000 UTC m=+0.589255047 container init 1e6261e53b5cf528827ce5035a0499cdb1ddfe121796f70e085ff4701b0bfc8a (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-2-activate, org.label-schema.build-date=20240319, io.buildah.version=1.33.5, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-03-21T12:43:17.078 INFO:journalctl@ceph.osd.2.smithi022.stdout:Mar 21 12:43:16 smithi022 podman[77934]: 2024-03-21 12:43:16.855222492 +0000 UTC m=+0.601067252 container start 1e6261e53b5cf528827ce5035a0499cdb1ddfe121796f70e085ff4701b0bfc8a (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-2-activate, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240319, GIT_CLEAN=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux ) 2024-03-21T12:43:17.078 INFO:journalctl@ceph.osd.2.smithi022.stdout:Mar 21 12:43:17 smithi022 podman[77934]: 2024-03-21 12:43:17.019190401 +0000 UTC m=+0.765035163 container attach 1e6261e53b5cf528827ce5035a0499cdb1ddfe121796f70e085ff4701b0bfc8a (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-2-activate, GIT_BRANCH=HEAD, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240319, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-21T12:43:17.585 INFO:journalctl@ceph.osd.2.smithi022.stdout:Mar 21 12:43:17 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-2-activate[77947]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-03-21T12:43:17.585 INFO:journalctl@ceph.osd.2.smithi022.stdout:Mar 21 12:43:17 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-2-activate[77947]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-03-21T12:43:17.585 INFO:journalctl@ceph.osd.2.smithi022.stdout:Mar 21 12:43:17 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-2-activate[77947]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-03-21T12:43:17.586 INFO:journalctl@ceph.osd.2.smithi022.stdout:Mar 21 12:43:17 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-2-activate[77947]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-03-21T12:43:17.586 INFO:journalctl@ceph.osd.2.smithi022.stdout:Mar 21 12:43:17 smithi022 bash[77934]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-03-21T12:43:17.586 INFO:journalctl@ceph.osd.2.smithi022.stdout:Mar 21 12:43:17 smithi022 bash[77934]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-03-21T12:43:17.586 INFO:journalctl@ceph.osd.2.smithi022.stdout:Mar 21 12:43:17 smithi022 bash[77934]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-03-21T12:43:17.586 INFO:journalctl@ceph.osd.2.smithi022.stdout:Mar 21 12:43:17 smithi022 bash[77934]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-03-21T12:43:17.586 INFO:journalctl@ceph.osd.2.smithi022.stdout:Mar 21 12:43:17 smithi022 bash[77934]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-03-21T12:43:17.586 INFO:journalctl@ceph.osd.2.smithi022.stdout:Mar 21 12:43:17 smithi022 bash[77934]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-03-21T12:43:17.586 INFO:journalctl@ceph.osd.2.smithi022.stdout:Mar 21 12:43:17 smithi022 bash[77934]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-03-21T12:43:17.586 INFO:journalctl@ceph.osd.2.smithi022.stdout:Mar 21 12:43:17 smithi022 bash[77934]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-03-21T12:43:17.586 INFO:journalctl@ceph.osd.2.smithi022.stdout:Mar 21 12:43:17 smithi022 bash[77934]: --> ceph-volume raw activate successful for osd ID: 2 2024-03-21T12:43:17.586 INFO:journalctl@ceph.osd.2.smithi022.stdout:Mar 21 12:43:17 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-2-activate[77947]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-03-21T12:43:17.586 INFO:journalctl@ceph.osd.2.smithi022.stdout:Mar 21 12:43:17 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-2-activate[77947]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-03-21T12:43:17.586 INFO:journalctl@ceph.osd.2.smithi022.stdout:Mar 21 12:43:17 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-2-activate[77947]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-03-21T12:43:17.586 INFO:journalctl@ceph.osd.2.smithi022.stdout:Mar 21 12:43:17 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-2-activate[77947]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-03-21T12:43:17.587 INFO:journalctl@ceph.osd.2.smithi022.stdout:Mar 21 12:43:17 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-2-activate[77947]: --> ceph-volume raw activate successful for osd ID: 2 2024-03-21T12:43:17.835 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:17 smithi022 ceph-mon[67047]: pgmap v133: 97 pgs: 6 active+undersized, 17 active+undersized+degraded, 3 peering, 71 active+clean; 345 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.9 MiB/s wr, 500 op/s; 133/1617 objects degraded (8.225%) 2024-03-21T12:43:17.836 INFO:journalctl@ceph.osd.2.smithi022.stdout:Mar 21 12:43:17 smithi022 podman[77934]: 2024-03-21 12:43:17.618053309 +0000 UTC m=+1.363898067 container died 1e6261e53b5cf528827ce5035a0499cdb1ddfe121796f70e085ff4701b0bfc8a (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-2-activate, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240319, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-21T12:43:17.879 INFO:teuthology.orchestra.run.smithi022.stdout: 5 224028 10.73 MB/sec execute 377 sec latency 35.656 ms 2024-03-21T12:43:18.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:17 smithi189 ceph-mon[53160]: pgmap v133: 97 pgs: 6 active+undersized, 17 active+undersized+degraded, 3 peering, 71 active+clean; 345 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.9 MiB/s wr, 500 op/s; 133/1617 objects degraded (8.225%) 2024-03-21T12:43:18.825 INFO:journalctl@ceph.osd.2.smithi022.stdout:Mar 21 12:43:18 smithi022 podman[78089]: 2024-03-21 12:43:18.670116795 +0000 UTC m=+1.043722138 container remove 1e6261e53b5cf528827ce5035a0499cdb1ddfe121796f70e085ff4701b0bfc8a (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-2-activate, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, GIT_CLEAN=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240319, org.label-schema.license=GPLv2) 2024-03-21T12:43:18.879 INFO:teuthology.orchestra.run.smithi022.stdout: 5 224605 10.76 MB/sec execute 378 sec latency 39.510 ms 2024-03-21T12:43:19.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:18 smithi022 ceph-mon[67047]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-03-21T12:43:19.095 INFO:journalctl@ceph.osd.2.smithi022.stdout:Mar 21 12:43:18 smithi022 podman[78171]: 2024-03-21 12:43:18.832639958 +0000 UTC m=+0.027123577 image pull 19e12293512ff7978c8adb5f72a829060dd47f07bfd6903b6b3178d2ebc29e4e quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93 2024-03-21T12:43:19.095 INFO:journalctl@ceph.osd.2.smithi022.stdout:Mar 21 12:43:18 smithi022 podman[78171]: 2024-03-21 12:43:18.9713713 +0000 UTC m=+0.165854899 container create 78a229f593f848a2c11074bac4d2ef708806f2c16fda03dd345892e5718d3395 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, ceph=True, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240319, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-03-21T12:43:19.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:18 smithi189 ceph-mon[53160]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-03-21T12:43:19.346 INFO:journalctl@ceph.osd.2.smithi022.stdout:Mar 21 12:43:19 smithi022 podman[78171]: 2024-03-21 12:43:19.291910225 +0000 UTC m=+0.486393849 container init 78a229f593f848a2c11074bac4d2ef708806f2c16fda03dd345892e5718d3395 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.build-date=20240319, ceph=True, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-03-21T12:43:19.347 INFO:journalctl@ceph.osd.2.smithi022.stdout:Mar 21 12:43:19 smithi022 podman[78171]: 2024-03-21 12:43:19.298710842 +0000 UTC m=+0.493194448 container start 78a229f593f848a2c11074bac4d2ef708806f2c16fda03dd345892e5718d3395 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.build-date=20240319, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS) 2024-03-21T12:43:19.347 INFO:journalctl@ceph.osd.2.smithi022.stdout:Mar 21 12:43:19 smithi022 ceph-osd[78189]: -- 172.21.15.22:0/517382322 <== mon.0 v2:172.21.15.22:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5632ffd0b680 con 0x5632ffcd4c00 2024-03-21T12:43:19.597 INFO:journalctl@ceph.osd.2.smithi022.stdout:Mar 21 12:43:19 smithi022 bash[78171]: 78a229f593f848a2c11074bac4d2ef708806f2c16fda03dd345892e5718d3395 2024-03-21T12:43:19.597 INFO:journalctl@ceph.osd.2.smithi022.stdout:Mar 21 12:43:19 smithi022 systemd[1]: Started Ceph osd.2 for 3fc7f0f2-e77e-11ee-95cd-87774f69a715. 2024-03-21T12:43:19.879 INFO:teuthology.orchestra.run.smithi022.stdout: 5 225187 10.77 MB/sec execute 379 sec latency 35.193 ms 2024-03-21T12:43:20.099 INFO:journalctl@ceph.osd.2.smithi022.stdout:Mar 21 12:43:19 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-2[78185]: 2024-03-21T12:43:19.881+0000 7fe33f0b67c0 -1 Falling back to public interface 2024-03-21T12:43:20.100 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:19 smithi022 ceph-mon[67047]: pgmap v134: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 346 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.7 MiB/s wr, 625 op/s; 158/1641 objects degraded (9.628%) 2024-03-21T12:43:20.100 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:19 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:43:20.100 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:19 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:43:20.100 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:19 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:43:20.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:19 smithi189 ceph-mon[53160]: pgmap v134: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 346 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.7 MiB/s wr, 625 op/s; 158/1641 objects degraded (9.628%) 2024-03-21T12:43:20.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:19 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:43:20.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:19 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:43:20.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:19 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:43:20.879 INFO:teuthology.orchestra.run.smithi022.stdout: 5 225663 10.78 MB/sec execute 380 sec latency 27.341 ms 2024-03-21T12:43:21.879 INFO:teuthology.orchestra.run.smithi022.stdout: 5 226118 10.79 MB/sec execute 381 sec latency 43.007 ms 2024-03-21T12:43:21.892 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:21 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:43:21.893 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:21 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:43:21.893 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:21 smithi022 ceph-mon[67047]: pgmap v135: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 346 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 3.9 KiB/s rd, 2.7 MiB/s wr, 312 op/s; 158/1641 objects degraded (9.628%) 2024-03-21T12:43:22.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:21 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:43:22.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:21 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:43:22.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:21 smithi189 ceph-mon[53160]: pgmap v135: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 346 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 3.9 KiB/s rd, 2.7 MiB/s wr, 312 op/s; 158/1641 objects degraded (9.628%) 2024-03-21T12:43:22.879 INFO:teuthology.orchestra.run.smithi022.stdout: 5 226405 10.78 MB/sec execute 382 sec latency 47.584 ms 2024-03-21T12:43:22.914 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:22 smithi022 ceph-mon[67047]: Health check update: Degraded data redundancy: 158/1752 objects degraded (9.018%), 19 pgs degraded (PG_DEGRADED) 2024-03-21T12:43:23.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:22 smithi189 ceph-mon[53160]: Health check update: Degraded data redundancy: 158/1752 objects degraded (9.018%), 19 pgs degraded (PG_DEGRADED) 2024-03-21T12:43:23.879 INFO:teuthology.orchestra.run.smithi022.stdout: 5 226679 10.77 MB/sec execute 383 sec latency 88.664 ms 2024-03-21T12:43:23.921 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:23 smithi022 ceph-mon[67047]: pgmap v136: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 349 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.1 MiB/s wr, 480 op/s; 158/1752 objects degraded (9.018%) 2024-03-21T12:43:24.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:23 smithi189 ceph-mon[53160]: pgmap v136: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 349 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.1 MiB/s wr, 480 op/s; 158/1752 objects degraded (9.018%) 2024-03-21T12:43:24.879 INFO:teuthology.orchestra.run.smithi022.stdout: 5 227023 10.76 MB/sec execute 384 sec latency 65.552 ms 2024-03-21T12:43:24.959 INFO:journalctl@ceph.osd.2.smithi022.stdout:Mar 21 12:43:24 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-2[78185]: 2024-03-21T12:43:24.916+0000 7fe33f0b67c0 -1 osd.2 0 read_superblock omap replica is missing. 2024-03-21T12:43:25.879 INFO:teuthology.orchestra.run.smithi022.stdout: 5 227264 10.74 MB/sec execute 385 sec latency 75.173 ms 2024-03-21T12:43:26.003 INFO:journalctl@ceph.osd.2.smithi022.stdout:Mar 21 12:43:25 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-2[78185]: 2024-03-21T12:43:25.996+0000 7fe33f0b67c0 -1 osd.2 99 log_to_monitors true 2024-03-21T12:43:26.004 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:25 smithi022 ceph-mon[67047]: pgmap v137: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 354 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.8 MiB/s wr, 503 op/s; 153/1746 objects degraded (8.763%) 2024-03-21T12:43:26.004 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:25 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:43:26.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:25 smithi189 ceph-mon[53160]: pgmap v137: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 354 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.8 MiB/s wr, 503 op/s; 153/1746 objects degraded (8.763%) 2024-03-21T12:43:26.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:25 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:43:26.879 INFO:teuthology.orchestra.run.smithi022.stdout: 5 227467 10.72 MB/sec execute 386 sec latency 70.809 ms 2024-03-21T12:43:27.047 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:26 smithi022 ceph-mon[67047]: from='osd.2 [v2:172.21.15.22:6818/670823495,v1:172.21.15.22:6819/670823495]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-03-21T12:43:27.047 INFO:journalctl@ceph.osd.2.smithi022.stdout:Mar 21 12:43:26 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-2[78185]: 2024-03-21T12:43:26.964+0000 7fe33050e700 -1 osd.2 99 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-21T12:43:27.110 INFO:teuthology.orchestra.run.smithi022.stdout:true 2024-03-21T12:43:27.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:26 smithi189 ceph-mon[53160]: from='osd.2 [v2:172.21.15.22:6818/670823495,v1:172.21.15.22:6819/670823495]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-03-21T12:43:27.879 INFO:teuthology.orchestra.run.smithi022.stdout: 5 227696 10.71 MB/sec execute 387 sec latency 69.229 ms 2024-03-21T12:43:27.909 INFO:teuthology.orchestra.run.smithi022.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-21T12:43:27.909 INFO:teuthology.orchestra.run.smithi022.stdout:alertmanager.smithi022 smithi022 *:9093,9094 running (5m) 33s ago 14m 24.1M - 0.23.0 ba2b418f427c 14eb2a3ca318 2024-03-21T12:43:27.909 INFO:teuthology.orchestra.run.smithi022.stdout:crash.smithi022 smithi022 running (2m) 33s ago 14m 6660k - 19.0.0-2267-g960fbbe4 19e12293512f 356b2ddb609a 2024-03-21T12:43:27.909 INFO:teuthology.orchestra.run.smithi022.stdout:crash.smithi189 smithi189 running (2m) 116s ago 14m 6660k - 19.0.0-2267-g960fbbe4 19e12293512f 90ee837f0500 2024-03-21T12:43:27.909 INFO:teuthology.orchestra.run.smithi022.stdout:grafana.smithi022 smithi022 *:3000 error 33s ago 14m - - 2024-03-21T12:43:27.909 INFO:teuthology.orchestra.run.smithi022.stdout:mds.foofs.smithi022.bcnxho smithi022 running (9m) 33s ago 9m 38.1M - 17.2.0 e1d6a67b021e db26b67b8ee7 2024-03-21T12:43:27.909 INFO:teuthology.orchestra.run.smithi022.stdout:mds.foofs.smithi189.lluvko smithi189 running (8m) 116s ago 8m 16.9M - 17.2.0 e1d6a67b021e cf153aff1387 2024-03-21T12:43:27.909 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.smithi022.fektzs smithi022 *:8443,9283,8765 running (4m) 33s ago 16m 503M - 19.0.0-2267-g960fbbe4 19e12293512f d57cebd306b3 2024-03-21T12:43:27.909 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.smithi189.cnxzvu smithi189 *:8443,9283,8765 running (3m) 116s ago 14m 450M - 19.0.0-2267-g960fbbe4 19e12293512f 41663955c56f 2024-03-21T12:43:27.909 INFO:teuthology.orchestra.run.smithi022.stdout:mon.smithi022 smithi022 running (3m) 33s ago 16m 48.2M 2048M 19.0.0-2267-g960fbbe4 19e12293512f c620c9071b2c 2024-03-21T12:43:27.910 INFO:teuthology.orchestra.run.smithi022.stdout:mon.smithi189 smithi189 running (3m) 116s ago 14m 35.8M 2048M 19.0.0-2267-g960fbbe4 19e12293512f b353af3d8f3b 2024-03-21T12:43:27.910 INFO:teuthology.orchestra.run.smithi022.stdout:nfs.foo.0.0.smithi022.ynhbbq smithi022 *:2049 running (8m) 33s ago 8m 276M - 4.0 e1d6a67b021e 33e48e64c60b 2024-03-21T12:43:27.910 INFO:teuthology.orchestra.run.smithi022.stdout:nfs.foo.1.0.smithi189.yzsqml smithi189 *:2049 running (8m) 116s ago 8m 85.4M - 4.0 e1d6a67b021e 48fab2cbdde2 2024-03-21T12:43:27.910 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.smithi022 smithi022 *:9100 running (5m) 33s ago 14m 21.0M - 1.3.1 1dbe0e931976 33287f803f4f 2024-03-21T12:43:27.910 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.smithi189 smithi189 *:9100 running (5m) 116s ago 13m 17.5M - 1.3.1 1dbe0e931976 b51d872e438a 2024-03-21T12:43:27.910 INFO:teuthology.orchestra.run.smithi022.stdout:osd.0 smithi022 running (89s) 33s ago 13m 236M 1937M 19.0.0-2267-g960fbbe4 19e12293512f 23a85645eeea 2024-03-21T12:43:27.910 INFO:teuthology.orchestra.run.smithi022.stdout:osd.1 smithi022 running (43s) 33s ago 12m 12.2M 1937M 19.0.0-2267-g960fbbe4 19e12293512f 0f90e70bbf0a 2024-03-21T12:43:27.910 INFO:teuthology.orchestra.run.smithi022.stdout:osd.2 smithi022 starting - - - 1937M 2024-03-21T12:43:27.910 INFO:teuthology.orchestra.run.smithi022.stdout:osd.3 smithi022 running (12m) 33s ago 12m 386M 1937M 17.2.0 e1d6a67b021e b1462ffc1b9e 2024-03-21T12:43:27.910 INFO:teuthology.orchestra.run.smithi022.stdout:osd.4 smithi189 running (11m) 116s ago 11m 560M 2704M 17.2.0 e1d6a67b021e 1bedc4eb02ed 2024-03-21T12:43:27.910 INFO:teuthology.orchestra.run.smithi022.stdout:osd.5 smithi189 running (11m) 116s ago 11m 407M 2704M 17.2.0 e1d6a67b021e 25fbab9786a5 2024-03-21T12:43:27.910 INFO:teuthology.orchestra.run.smithi022.stdout:osd.6 smithi189 running (10m) 116s ago 10m 578M 2704M 17.2.0 e1d6a67b021e 0c192fcfd65b 2024-03-21T12:43:27.910 INFO:teuthology.orchestra.run.smithi022.stdout:osd.7 smithi189 running (10m) 116s ago 10m 528M 2704M 17.2.0 e1d6a67b021e 3d0723e1f863 2024-03-21T12:43:27.910 INFO:teuthology.orchestra.run.smithi022.stdout:prometheus.smithi022 smithi022 *:9095 running (4m) 33s ago 13m 73.2M - 2.33.4 514e6a882f6e 8171bbe41d2c 2024-03-21T12:43:28.085 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:27 smithi022 ceph-mon[67047]: pgmap v138: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 342 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 10 KiB/s rd, 5.6 MiB/s wr, 595 op/s; 144/1605 objects degraded (8.972%) 2024-03-21T12:43:28.085 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:27 smithi022 ceph-mon[67047]: from='osd.2 [v2:172.21.15.22:6818/670823495,v1:172.21.15.22:6819/670823495]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-03-21T12:43:28.085 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:27 smithi022 ceph-mon[67047]: osdmap e102: 8 total, 7 up, 8 in 2024-03-21T12:43:28.085 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:27 smithi022 ceph-mon[67047]: from='osd.2 [v2:172.21.15.22:6818/670823495,v1:172.21.15.22:6819/670823495]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]: dispatch 2024-03-21T12:43:28.085 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:27 smithi022 ceph-mon[67047]: from='client.44199 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:43:28.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:27 smithi189 ceph-mon[53160]: pgmap v138: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 342 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 10 KiB/s rd, 5.6 MiB/s wr, 595 op/s; 144/1605 objects degraded (8.972%) 2024-03-21T12:43:28.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:27 smithi189 ceph-mon[53160]: from='osd.2 [v2:172.21.15.22:6818/670823495,v1:172.21.15.22:6819/670823495]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-03-21T12:43:28.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:27 smithi189 ceph-mon[53160]: osdmap e102: 8 total, 7 up, 8 in 2024-03-21T12:43:28.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:27 smithi189 ceph-mon[53160]: from='osd.2 [v2:172.21.15.22:6818/670823495,v1:172.21.15.22:6819/670823495]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]: dispatch 2024-03-21T12:43:28.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:27 smithi189 ceph-mon[53160]: from='client.44199 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:43:28.399 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-03-21T12:43:28.399 INFO:teuthology.orchestra.run.smithi022.stdout: "mon": { 2024-03-21T12:43:28.399 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 2 2024-03-21T12:43:28.399 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:43:28.399 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr": { 2024-03-21T12:43:28.399 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 2 2024-03-21T12:43:28.399 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:43:28.399 INFO:teuthology.orchestra.run.smithi022.stdout: "osd": { 2024-03-21T12:43:28.399 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-03-21T12:43:28.400 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 3 2024-03-21T12:43:28.400 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:43:28.400 INFO:teuthology.orchestra.run.smithi022.stdout: "mds": { 2024-03-21T12:43:28.400 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-21T12:43:28.400 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:43:28.400 INFO:teuthology.orchestra.run.smithi022.stdout: "overall": { 2024-03-21T12:43:28.400 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-03-21T12:43:28.400 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 7 2024-03-21T12:43:28.400 INFO:teuthology.orchestra.run.smithi022.stdout: } 2024-03-21T12:43:28.400 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-03-21T12:43:28.747 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-03-21T12:43:28.747 INFO:teuthology.orchestra.run.smithi022.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93", 2024-03-21T12:43:28.748 INFO:teuthology.orchestra.run.smithi022.stdout: "in_progress": true, 2024-03-21T12:43:28.748 INFO:teuthology.orchestra.run.smithi022.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-21T12:43:28.748 INFO:teuthology.orchestra.run.smithi022.stdout: "services_complete": [ 2024-03-21T12:43:28.748 INFO:teuthology.orchestra.run.smithi022.stdout: "mon", 2024-03-21T12:43:28.748 INFO:teuthology.orchestra.run.smithi022.stdout: "crash", 2024-03-21T12:43:28.748 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr" 2024-03-21T12:43:28.748 INFO:teuthology.orchestra.run.smithi022.stdout: ], 2024-03-21T12:43:28.748 INFO:teuthology.orchestra.run.smithi022.stdout: "progress": "8/23 daemons upgraded", 2024-03-21T12:43:28.748 INFO:teuthology.orchestra.run.smithi022.stdout: "message": "Currently upgrading osd daemons", 2024-03-21T12:43:28.748 INFO:teuthology.orchestra.run.smithi022.stdout: "is_paused": false 2024-03-21T12:43:28.748 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-03-21T12:43:28.879 INFO:teuthology.orchestra.run.smithi022.stdout: 5 227784 10.70 MB/sec execute 388 sec latency 933.433 ms 2024-03-21T12:43:28.949 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:28 smithi022 ceph-mon[67047]: from='client.34286 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:43:28.950 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:28 smithi022 ceph-mon[67047]: from='client.34290 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:43:28.950 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:28 smithi022 ceph-mon[67047]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-21T12:43:28.950 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:28 smithi022 ceph-mon[67047]: osd.2 [v2:172.21.15.22:6818/670823495,v1:172.21.15.22:6819/670823495] boot 2024-03-21T12:43:28.950 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:28 smithi022 ceph-mon[67047]: osdmap e103: 8 total, 8 up, 8 in 2024-03-21T12:43:28.950 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:28 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-21T12:43:28.950 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:28 smithi022 ceph-mon[67047]: from='client.? 172.21.15.22:0/752771876' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:43:28.950 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:28 smithi022 ceph-mon[67047]: Health check update: Degraded data redundancy: 144/1605 objects degraded (8.972%), 19 pgs degraded (PG_DEGRADED) 2024-03-21T12:43:28.950 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:28 smithi022 ceph-mon[67047]: osdmap e104: 8 total, 8 up, 8 in 2024-03-21T12:43:29.186 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:28 smithi189 ceph-mon[53160]: from='client.34286 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:43:29.186 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:28 smithi189 ceph-mon[53160]: from='client.34290 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:43:29.186 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:28 smithi189 ceph-mon[53160]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-21T12:43:29.187 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:28 smithi189 ceph-mon[53160]: osd.2 [v2:172.21.15.22:6818/670823495,v1:172.21.15.22:6819/670823495] boot 2024-03-21T12:43:29.187 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:28 smithi189 ceph-mon[53160]: osdmap e103: 8 total, 8 up, 8 in 2024-03-21T12:43:29.187 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:28 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-21T12:43:29.187 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:28 smithi189 ceph-mon[53160]: from='client.? 172.21.15.22:0/752771876' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:43:29.187 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:28 smithi189 ceph-mon[53160]: Health check update: Degraded data redundancy: 144/1605 objects degraded (8.972%), 19 pgs degraded (PG_DEGRADED) 2024-03-21T12:43:29.187 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:28 smithi189 ceph-mon[53160]: osdmap e104: 8 total, 8 up, 8 in 2024-03-21T12:43:29.268 INFO:teuthology.orchestra.run.smithi022.stdout:HEALTH_WARN 1 failed cephadm daemon(s); Degraded data redundancy: 144/1605 objects degraded (8.972%), 19 pgs degraded 2024-03-21T12:43:29.268 INFO:teuthology.orchestra.run.smithi022.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-21T12:43:29.268 INFO:teuthology.orchestra.run.smithi022.stdout: daemon grafana.smithi022 on smithi022 is in error state 2024-03-21T12:43:29.268 INFO:teuthology.orchestra.run.smithi022.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 144/1605 objects degraded (8.972%), 19 pgs degraded 2024-03-21T12:43:29.268 INFO:teuthology.orchestra.run.smithi022.stdout: pg 2.1 is active+undersized+degraded, acting [3,0] 2024-03-21T12:43:29.269 INFO:teuthology.orchestra.run.smithi022.stdout: pg 2.3 is active+undersized+degraded, acting [5,7] 2024-03-21T12:43:29.269 INFO:teuthology.orchestra.run.smithi022.stdout: pg 2.7 is active+undersized+degraded, acting [6,7] 2024-03-21T12:43:29.269 INFO:teuthology.orchestra.run.smithi022.stdout: pg 2.8 is active+undersized+degraded, acting [0,1] 2024-03-21T12:43:29.269 INFO:teuthology.orchestra.run.smithi022.stdout: pg 2.c is active+undersized+degraded, acting [5,0] 2024-03-21T12:43:29.269 INFO:teuthology.orchestra.run.smithi022.stdout: pg 2.e is active+undersized+degraded, acting [3,7] 2024-03-21T12:43:29.269 INFO:teuthology.orchestra.run.smithi022.stdout: pg 2.10 is active+undersized+degraded, acting [1,0] 2024-03-21T12:43:29.269 INFO:teuthology.orchestra.run.smithi022.stdout: pg 2.16 is active+undersized+degraded, acting [5,6] 2024-03-21T12:43:29.269 INFO:teuthology.orchestra.run.smithi022.stdout: pg 2.17 is active+undersized+degraded, acting [6,5] 2024-03-21T12:43:29.269 INFO:teuthology.orchestra.run.smithi022.stdout: pg 2.1c is active+undersized+degraded, acting [7,5] 2024-03-21T12:43:29.269 INFO:teuthology.orchestra.run.smithi022.stdout: pg 3.0 is active+undersized+degraded, acting [1,6] 2024-03-21T12:43:29.269 INFO:teuthology.orchestra.run.smithi022.stdout: pg 3.4 is active+undersized+degraded, acting [1,5] 2024-03-21T12:43:29.269 INFO:teuthology.orchestra.run.smithi022.stdout: pg 3.5 is active+undersized+degraded, acting [5,3] 2024-03-21T12:43:29.269 INFO:teuthology.orchestra.run.smithi022.stdout: pg 3.9 is active+undersized+degraded, acting [4,7] 2024-03-21T12:43:29.269 INFO:teuthology.orchestra.run.smithi022.stdout: pg 3.13 is active+undersized+degraded, acting [7,4] 2024-03-21T12:43:29.269 INFO:teuthology.orchestra.run.smithi022.stdout: pg 3.1a is active+undersized+degraded, acting [4,1] 2024-03-21T12:43:29.270 INFO:teuthology.orchestra.run.smithi022.stdout: pg 3.1e is active+undersized+degraded, acting [3,6] 2024-03-21T12:43:29.270 INFO:teuthology.orchestra.run.smithi022.stdout: pg 3.1f is active+undersized+degraded, acting [0,5] 2024-03-21T12:43:29.270 INFO:teuthology.orchestra.run.smithi022.stdout: pg 4.19 is active+undersized+degraded, acting [3,0] 2024-03-21T12:43:29.879 INFO:teuthology.orchestra.run.smithi022.stdout: 5 227895 10.68 MB/sec execute 389 sec latency 1870.121 ms 2024-03-21T12:43:30.088 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:29 smithi022 ceph-mon[67047]: from='client.34298 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:43:30.089 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:29 smithi022 ceph-mon[67047]: pgmap v142: 97 pgs: 4 peering, 5 active+undersized, 17 active+undersized+degraded, 71 active+clean; 339 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 3.8 MiB/s wr, 481 op/s; 115/1554 objects degraded (7.400%) 2024-03-21T12:43:30.089 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:29 smithi022 ceph-mon[67047]: from='client.? 172.21.15.22:0/2636610083' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-21T12:43:30.089 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:29 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:43:30.089 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:29 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:43:30.089 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:29 smithi022 ceph-mon[67047]: osdmap e105: 8 total, 8 up, 8 in 2024-03-21T12:43:30.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:29 smithi189 ceph-mon[53160]: from='client.34298 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:43:30.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:29 smithi189 ceph-mon[53160]: pgmap v142: 97 pgs: 4 peering, 5 active+undersized, 17 active+undersized+degraded, 71 active+clean; 339 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 3.8 MiB/s wr, 481 op/s; 115/1554 objects degraded (7.400%) 2024-03-21T12:43:30.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:29 smithi189 ceph-mon[53160]: from='client.? 172.21.15.22:0/2636610083' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-21T12:43:30.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:29 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:43:30.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:29 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:43:30.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:29 smithi189 ceph-mon[53160]: osdmap e105: 8 total, 8 up, 8 in 2024-03-21T12:43:30.882 INFO:teuthology.orchestra.run.smithi022.stdout: 5 228200 10.68 MB/sec execute 390 sec latency 49.313 ms 2024-03-21T12:43:31.882 INFO:teuthology.orchestra.run.smithi022.stdout: 5 228531 10.68 MB/sec execute 391 sec latency 553.965 ms 2024-03-21T12:43:32.104 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:31 smithi022 ceph-mon[67047]: pgmap v144: 97 pgs: 4 peering, 5 active+undersized, 17 active+undersized+degraded, 71 active+clean; 339 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 1.3 MiB/s wr, 132 op/s; 115/1554 objects degraded (7.400%) 2024-03-21T12:43:32.104 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:31 smithi022 ceph-mon[67047]: osdmap e106: 8 total, 8 up, 8 in 2024-03-21T12:43:32.104 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:31 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:43:32.104 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:31 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:43:32.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:31 smithi189 ceph-mon[53160]: pgmap v144: 97 pgs: 4 peering, 5 active+undersized, 17 active+undersized+degraded, 71 active+clean; 339 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 1.3 MiB/s wr, 132 op/s; 115/1554 objects degraded (7.400%) 2024-03-21T12:43:32.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:31 smithi189 ceph-mon[53160]: osdmap e106: 8 total, 8 up, 8 in 2024-03-21T12:43:32.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:31 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:43:32.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:31 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:43:32.882 INFO:teuthology.orchestra.run.smithi022.stdout: 5 228801 10.67 MB/sec execute 392 sec latency 634.994 ms 2024-03-21T12:43:33.134 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:32 smithi022 ceph-mon[67047]: osdmap e107: 8 total, 8 up, 8 in 2024-03-21T12:43:33.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:32 smithi189 ceph-mon[53160]: osdmap e107: 8 total, 8 up, 8 in 2024-03-21T12:43:33.882 INFO:teuthology.orchestra.run.smithi022.stdout: 5 229149 10.66 MB/sec execute 393 sec latency 606.454 ms 2024-03-21T12:43:34.157 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:33 smithi022 ceph-mon[67047]: pgmap v147: 97 pgs: 5 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 90 active+clean; 336 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 3.0 KiB/s rd, 3.7 MiB/s wr, 350 op/s; 27/1581 objects degraded (1.708%); 74 KiB/s, 28 objects/s recovering 2024-03-21T12:43:34.158 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:33 smithi022 ceph-mon[67047]: osdmap e108: 8 total, 8 up, 8 in 2024-03-21T12:43:34.158 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:33 smithi022 ceph-mon[67047]: Health check update: Degraded data redundancy: 27/1581 objects degraded (1.708%), 2 pgs degraded (PG_DEGRADED) 2024-03-21T12:43:34.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:33 smithi189 ceph-mon[53160]: pgmap v147: 97 pgs: 5 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 90 active+clean; 336 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 3.0 KiB/s rd, 3.7 MiB/s wr, 350 op/s; 27/1581 objects degraded (1.708%); 74 KiB/s, 28 objects/s recovering 2024-03-21T12:43:34.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:33 smithi189 ceph-mon[53160]: osdmap e108: 8 total, 8 up, 8 in 2024-03-21T12:43:34.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:33 smithi189 ceph-mon[53160]: Health check update: Degraded data redundancy: 27/1581 objects degraded (1.708%), 2 pgs degraded (PG_DEGRADED) 2024-03-21T12:43:34.882 INFO:teuthology.orchestra.run.smithi022.stdout: 5 229370 10.64 MB/sec execute 394 sec latency 66.093 ms 2024-03-21T12:43:35.182 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:34 smithi022 ceph-mon[67047]: osdmap e109: 8 total, 8 up, 8 in 2024-03-21T12:43:35.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:34 smithi189 ceph-mon[53160]: osdmap e109: 8 total, 8 up, 8 in 2024-03-21T12:43:35.882 INFO:teuthology.orchestra.run.smithi022.stdout: 5 229594 10.63 MB/sec execute 395 sec latency 61.052 ms 2024-03-21T12:43:36.197 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:35 smithi022 ceph-mon[67047]: pgmap v150: 97 pgs: 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 94 active+clean; 338 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 3.0 KiB/s rd, 4.2 MiB/s wr, 373 op/s; 27/1614 objects degraded (1.673%); 133 KiB/s, 37 objects/s recovering 2024-03-21T12:43:36.198 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:35 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:43:36.198 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:35 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:43:36.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:35 smithi189 ceph-mon[53160]: pgmap v150: 97 pgs: 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 94 active+clean; 338 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 3.0 KiB/s rd, 4.2 MiB/s wr, 373 op/s; 27/1614 objects degraded (1.673%); 133 KiB/s, 37 objects/s recovering 2024-03-21T12:43:36.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:35 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:43:36.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:35 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:43:36.882 INFO:teuthology.orchestra.run.smithi022.stdout: 5 229886 10.63 MB/sec execute 396 sec latency 70.173 ms 2024-03-21T12:43:37.756 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:37 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:43:37.756 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:37 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:43:37.756 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:37 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:43:37.756 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:37 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:43:37.756 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:37 smithi022 ceph-mon[67047]: pgmap v151: 97 pgs: 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 94 active+clean; 338 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 2.1 KiB/s rd, 3.0 MiB/s wr, 263 op/s; 27/1614 objects degraded (1.673%); 94 KiB/s, 26 objects/s recovering 2024-03-21T12:43:37.756 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:37 smithi022 ceph-mon[67047]: pgmap v152: 97 pgs: 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 94 active+clean; 338 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 895 KiB/s wr, 30 op/s; 27/1614 objects degraded (1.673%); 91 KiB/s, 12 objects/s recovering 2024-03-21T12:43:37.756 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:37 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:43:37.756 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:37 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:43:37.756 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:37 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:43:37.757 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:37 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:43:37.757 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:37 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:43:37.757 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:37 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:43:37.757 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:37 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-03-21T12:43:37.757 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:37 smithi022 ceph-mon[67047]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-03-21T12:43:37.757 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:37 smithi022 ceph-mon[67047]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-03-21T12:43:37.882 INFO:teuthology.orchestra.run.smithi022.stdout: 5 230162 10.62 MB/sec execute 397 sec latency 94.169 ms 2024-03-21T12:43:38.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:37 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:43:38.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:37 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:43:38.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:37 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:43:38.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:37 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:43:38.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:37 smithi189 ceph-mon[53160]: pgmap v151: 97 pgs: 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 94 active+clean; 338 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 2.1 KiB/s rd, 3.0 MiB/s wr, 263 op/s; 27/1614 objects degraded (1.673%); 94 KiB/s, 26 objects/s recovering 2024-03-21T12:43:38.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:37 smithi189 ceph-mon[53160]: pgmap v152: 97 pgs: 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 94 active+clean; 338 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 895 KiB/s wr, 30 op/s; 27/1614 objects degraded (1.673%); 91 KiB/s, 12 objects/s recovering 2024-03-21T12:43:38.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:37 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:43:38.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:37 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:43:38.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:37 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:43:38.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:37 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:43:38.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:37 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:43:38.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:37 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:43:38.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:37 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-03-21T12:43:38.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:37 smithi189 ceph-mon[53160]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-03-21T12:43:38.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:37 smithi189 ceph-mon[53160]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-03-21T12:43:38.767 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:38 smithi022 ceph-mon[67047]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 27/1614 objects degraded (1.673%), 2 pgs degraded) 2024-03-21T12:43:38.882 INFO:teuthology.orchestra.run.smithi022.stdout: 5 230609 10.62 MB/sec execute 398 sec latency 34.431 ms 2024-03-21T12:43:39.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:38 smithi189 ceph-mon[53160]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 27/1614 objects degraded (1.673%), 2 pgs degraded) 2024-03-21T12:43:39.777 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:39 smithi022 ceph-mon[67047]: pgmap v153: 97 pgs: 97 active+clean; 342 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.0 MiB/s wr, 407 op/s; 40 KiB/s, 10 objects/s recovering 2024-03-21T12:43:39.882 INFO:teuthology.orchestra.run.smithi022.stdout: 5 231028 10.62 MB/sec execute 399 sec latency 35.575 ms 2024-03-21T12:43:40.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:39 smithi189 ceph-mon[53160]: pgmap v153: 97 pgs: 97 active+clean; 342 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.0 MiB/s wr, 407 op/s; 40 KiB/s, 10 objects/s recovering 2024-03-21T12:43:40.882 INFO:teuthology.orchestra.run.smithi022.stdout: 5 231464 10.63 MB/sec execute 400 sec latency 49.235 ms 2024-03-21T12:43:41.835 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:41 smithi022 ceph-mon[67047]: pgmap v154: 97 pgs: 97 active+clean; 342 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 3.0 MiB/s wr, 308 op/s; 30 KiB/s, 8 objects/s recovering 2024-03-21T12:43:41.882 INFO:teuthology.orchestra.run.smithi022.stdout: 5 231857 10.64 MB/sec execute 401 sec latency 92.494 ms 2024-03-21T12:43:42.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:41 smithi189 ceph-mon[53160]: pgmap v154: 97 pgs: 97 active+clean; 342 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 3.0 MiB/s wr, 308 op/s; 30 KiB/s, 8 objects/s recovering 2024-03-21T12:43:42.882 INFO:teuthology.orchestra.run.smithi022.stdout: 5 232221 10.64 MB/sec execute 402 sec latency 53.487 ms 2024-03-21T12:43:43.851 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:43 smithi022 ceph-mon[67047]: pgmap v155: 97 pgs: 97 active+clean; 339 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 4.9 MiB/s wr, 398 op/s; 0 B/s, 3 objects/s recovering 2024-03-21T12:43:43.882 INFO:teuthology.orchestra.run.smithi022.stdout: 5 232510 10.62 MB/sec execute 403 sec latency 39.830 ms 2024-03-21T12:43:44.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:43 smithi189 ceph-mon[53160]: pgmap v155: 97 pgs: 97 active+clean; 339 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 4.9 MiB/s wr, 398 op/s; 0 B/s, 3 objects/s recovering 2024-03-21T12:43:44.882 INFO:teuthology.orchestra.run.smithi022.stdout: 5 232892 10.62 MB/sec execute 404 sec latency 41.577 ms 2024-03-21T12:43:45.846 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:45 smithi022 ceph-mon[67047]: pgmap v156: 97 pgs: 97 active+clean; 343 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.3 MiB/s wr, 556 op/s; 0 B/s, 2 objects/s recovering 2024-03-21T12:43:45.882 INFO:teuthology.orchestra.run.smithi022.stdout: 5 233226 10.62 MB/sec execute 405 sec latency 36.819 ms 2024-03-21T12:43:46.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:45 smithi189 ceph-mon[53160]: pgmap v156: 97 pgs: 97 active+clean; 343 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.3 MiB/s wr, 556 op/s; 0 B/s, 2 objects/s recovering 2024-03-21T12:43:46.882 INFO:teuthology.orchestra.run.smithi022.stdout: 5 233602 10.62 MB/sec execute 406 sec latency 46.222 ms 2024-03-21T12:43:46.904 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:46 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:43:47.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:46 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:43:47.883 INFO:teuthology.orchestra.run.smithi022.stdout: 5 234007 10.62 MB/sec execute 407 sec latency 52.196 ms 2024-03-21T12:43:47.925 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:47 smithi022 ceph-mon[67047]: pgmap v157: 97 pgs: 97 active+clean; 343 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.2 MiB/s wr, 545 op/s; 0 B/s, 2 objects/s recovering 2024-03-21T12:43:48.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:47 smithi189 ceph-mon[53160]: pgmap v157: 97 pgs: 97 active+clean; 343 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.2 MiB/s wr, 545 op/s; 0 B/s, 2 objects/s recovering 2024-03-21T12:43:48.882 INFO:teuthology.orchestra.run.smithi022.stdout: 5 234385 10.60 MB/sec execute 408 sec latency 45.329 ms 2024-03-21T12:43:49.883 INFO:teuthology.orchestra.run.smithi022.stdout: 5 234751 10.61 MB/sec execute 409 sec latency 42.007 ms 2024-03-21T12:43:49.954 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:49 smithi022 ceph-mon[67047]: pgmap v158: 97 pgs: 97 active+clean; 340 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.3 KiB/s rd, 6.3 MiB/s wr, 629 op/s; 0 B/s, 2 objects/s recovering 2024-03-21T12:43:50.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:49 smithi189 ceph-mon[53160]: pgmap v158: 97 pgs: 97 active+clean; 340 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.3 KiB/s rd, 6.3 MiB/s wr, 629 op/s; 0 B/s, 2 objects/s recovering 2024-03-21T12:43:50.883 INFO:teuthology.orchestra.run.smithi022.stdout: 5 235227 10.62 MB/sec execute 410 sec latency 37.517 ms 2024-03-21T12:43:51.731 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:51 smithi022 ceph-mon[67047]: pgmap v159: 97 pgs: 97 active+clean; 340 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.5 MiB/s wr, 439 op/s 2024-03-21T12:43:51.732 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:51 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:43:51.732 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:51 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:43:51.882 INFO:teuthology.orchestra.run.smithi022.stdout: 5 235661 10.63 MB/sec execute 411 sec latency 40.526 ms 2024-03-21T12:43:52.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:51 smithi189 ceph-mon[53160]: pgmap v159: 97 pgs: 97 active+clean; 340 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.5 MiB/s wr, 439 op/s 2024-03-21T12:43:52.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:51 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:43:52.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:51 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:43:52.883 INFO:teuthology.orchestra.run.smithi022.stdout: 5 236013 10.62 MB/sec execute 412 sec latency 44.505 ms 2024-03-21T12:43:52.888 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:52 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-03-21T12:43:52.888 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:52 smithi022 ceph-mon[67047]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-03-21T12:43:52.888 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:52 smithi022 ceph-mon[67047]: Upgrade: osd.3 is safe to restart 2024-03-21T12:43:53.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:52 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-03-21T12:43:53.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:52 smithi189 ceph-mon[53160]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-03-21T12:43:53.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:52 smithi189 ceph-mon[53160]: Upgrade: osd.3 is safe to restart 2024-03-21T12:43:53.883 INFO:teuthology.orchestra.run.smithi022.stdout: 5 236382 10.61 MB/sec execute 413 sec latency 35.024 ms 2024-03-21T12:43:53.895 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:53 smithi022 ceph-mon[67047]: pgmap v160: 97 pgs: 97 active+clean; 342 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 6.9 MiB/s wr, 676 op/s 2024-03-21T12:43:54.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:53 smithi189 ceph-mon[53160]: pgmap v160: 97 pgs: 97 active+clean; 342 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 6.9 MiB/s wr, 676 op/s 2024-03-21T12:43:54.883 INFO:teuthology.orchestra.run.smithi022.stdout: 5 236661 10.60 MB/sec execute 414 sec latency 136.935 ms 2024-03-21T12:43:55.883 INFO:teuthology.orchestra.run.smithi022.stdout: 5 236993 10.61 MB/sec execute 415 sec latency 55.944 ms 2024-03-21T12:43:56.474 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:56 smithi022 ceph-mon[67047]: pgmap v161: 97 pgs: 97 active+clean; 343 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 5.3 MiB/s wr, 588 op/s 2024-03-21T12:43:56.474 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:56 smithi022 ceph-mon[67047]: Upgrade: Updating osd.3 2024-03-21T12:43:56.474 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:56 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:43:56.474 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:56 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-03-21T12:43:56.474 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:56 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:43:56.474 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:56 smithi022 ceph-mon[67047]: Deploying daemon osd.3 on smithi022 2024-03-21T12:43:56.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:56 smithi189 ceph-mon[53160]: pgmap v161: 97 pgs: 97 active+clean; 343 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 5.3 MiB/s wr, 588 op/s 2024-03-21T12:43:56.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:56 smithi189 ceph-mon[53160]: Upgrade: Updating osd.3 2024-03-21T12:43:56.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:56 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:43:56.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:56 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-03-21T12:43:56.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:56 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:43:56.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:56 smithi189 ceph-mon[53160]: Deploying daemon osd.3 on smithi022 2024-03-21T12:43:56.883 INFO:teuthology.orchestra.run.smithi022.stdout: 5 237426 10.61 MB/sec execute 416 sec latency 64.636 ms 2024-03-21T12:43:57.883 INFO:teuthology.orchestra.run.smithi022.stdout: 5 237780 10.60 MB/sec execute 417 sec latency 39.766 ms 2024-03-21T12:43:58.491 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:43:58 smithi022 ceph-mon[67047]: pgmap v162: 97 pgs: 97 active+clean; 343 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.5 MiB/s wr, 421 op/s 2024-03-21T12:43:58.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:43:58 smithi189 ceph-mon[53160]: pgmap v162: 97 pgs: 97 active+clean; 343 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.5 MiB/s wr, 421 op/s 2024-03-21T12:43:58.883 INFO:teuthology.orchestra.run.smithi022.stdout: 5 238074 10.59 MB/sec execute 418 sec latency 44.583 ms 2024-03-21T12:43:59.793 INFO:teuthology.orchestra.run.smithi022.stdout:true 2024-03-21T12:43:59.883 INFO:teuthology.orchestra.run.smithi022.stdout: 5 238369 10.59 MB/sec execute 419 sec latency 51.275 ms 2024-03-21T12:44:00.533 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:00 smithi022 ceph-mon[67047]: pgmap v163: 97 pgs: 97 active+clean; 338 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.7 MiB/s wr, 689 op/s 2024-03-21T12:44:00.533 INFO:journalctl@ceph.osd.3.smithi022.stdout:Mar 21 12:44:00 smithi022 systemd[1]: Stopping Ceph osd.3 for 3fc7f0f2-e77e-11ee-95cd-87774f69a715... 2024-03-21T12:44:00.587 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:00 smithi189 ceph-mon[53160]: pgmap v163: 97 pgs: 97 active+clean; 338 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.7 MiB/s wr, 689 op/s 2024-03-21T12:44:00.817 INFO:teuthology.orchestra.run.smithi022.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-21T12:44:00.817 INFO:teuthology.orchestra.run.smithi022.stdout:alertmanager.smithi022 smithi022 *:9093,9094 running (6m) 31s ago 15m 24.1M - 0.23.0 ba2b418f427c 14eb2a3ca318 2024-03-21T12:44:00.817 INFO:teuthology.orchestra.run.smithi022.stdout:crash.smithi022 smithi022 running (2m) 31s ago 15m 6660k - 19.0.0-2267-g960fbbe4 19e12293512f 356b2ddb609a 2024-03-21T12:44:00.817 INFO:teuthology.orchestra.run.smithi022.stdout:crash.smithi189 smithi189 running (2m) 2m ago 14m 6660k - 19.0.0-2267-g960fbbe4 19e12293512f 90ee837f0500 2024-03-21T12:44:00.817 INFO:teuthology.orchestra.run.smithi022.stdout:grafana.smithi022 smithi022 *:3000 error 31s ago 15m - - 2024-03-21T12:44:00.817 INFO:teuthology.orchestra.run.smithi022.stdout:mds.foofs.smithi022.bcnxho smithi022 running (9m) 31s ago 9m 38.0M - 17.2.0 e1d6a67b021e db26b67b8ee7 2024-03-21T12:44:00.817 INFO:teuthology.orchestra.run.smithi022.stdout:mds.foofs.smithi189.lluvko smithi189 running (9m) 2m ago 9m 16.9M - 17.2.0 e1d6a67b021e cf153aff1387 2024-03-21T12:44:00.817 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.smithi022.fektzs smithi022 *:8443,9283,8765 running (5m) 31s ago 16m 505M - 19.0.0-2267-g960fbbe4 19e12293512f d57cebd306b3 2024-03-21T12:44:00.818 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.smithi189.cnxzvu smithi189 *:8443,9283,8765 running (4m) 2m ago 14m 450M - 19.0.0-2267-g960fbbe4 19e12293512f 41663955c56f 2024-03-21T12:44:00.818 INFO:teuthology.orchestra.run.smithi022.stdout:mon.smithi022 smithi022 running (4m) 31s ago 16m 49.4M 2048M 19.0.0-2267-g960fbbe4 19e12293512f c620c9071b2c 2024-03-21T12:44:00.818 INFO:teuthology.orchestra.run.smithi022.stdout:mon.smithi189 smithi189 running (3m) 2m ago 14m 35.8M 2048M 19.0.0-2267-g960fbbe4 19e12293512f b353af3d8f3b 2024-03-21T12:44:00.818 INFO:teuthology.orchestra.run.smithi022.stdout:nfs.foo.0.0.smithi022.ynhbbq smithi022 *:2049 running (9m) 31s ago 9m 274M - 4.0 e1d6a67b021e 33e48e64c60b 2024-03-21T12:44:00.818 INFO:teuthology.orchestra.run.smithi022.stdout:nfs.foo.1.0.smithi189.yzsqml smithi189 *:2049 running (9m) 2m ago 9m 85.4M - 4.0 e1d6a67b021e 48fab2cbdde2 2024-03-21T12:44:00.818 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.smithi022 smithi022 *:9100 running (6m) 31s ago 15m 20.7M - 1.3.1 1dbe0e931976 33287f803f4f 2024-03-21T12:44:00.818 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.smithi189 smithi189 *:9100 running (5m) 2m ago 14m 17.5M - 1.3.1 1dbe0e931976 b51d872e438a 2024-03-21T12:44:00.818 INFO:teuthology.orchestra.run.smithi022.stdout:osd.0 smithi022 running (2m) 31s ago 13m 285M 1937M 19.0.0-2267-g960fbbe4 19e12293512f 23a85645eeea 2024-03-21T12:44:00.818 INFO:teuthology.orchestra.run.smithi022.stdout:osd.1 smithi022 running (75s) 31s ago 13m 230M 1937M 19.0.0-2267-g960fbbe4 19e12293512f 0f90e70bbf0a 2024-03-21T12:44:00.818 INFO:teuthology.orchestra.run.smithi022.stdout:osd.2 smithi022 running (41s) 31s ago 12m 14.0M 1937M 19.0.0-2267-g960fbbe4 19e12293512f 78a229f593f8 2024-03-21T12:44:00.818 INFO:teuthology.orchestra.run.smithi022.stdout:osd.3 smithi022 running (12m) 31s ago 12m 403M 1937M 17.2.0 e1d6a67b021e b1462ffc1b9e 2024-03-21T12:44:00.818 INFO:teuthology.orchestra.run.smithi022.stdout:osd.4 smithi189 running (12m) 2m ago 12m 560M 2704M 17.2.0 e1d6a67b021e 1bedc4eb02ed 2024-03-21T12:44:00.818 INFO:teuthology.orchestra.run.smithi022.stdout:osd.5 smithi189 running (11m) 2m ago 11m 407M 2704M 17.2.0 e1d6a67b021e 25fbab9786a5 2024-03-21T12:44:00.819 INFO:teuthology.orchestra.run.smithi022.stdout:osd.6 smithi189 running (11m) 2m ago 11m 578M 2704M 17.2.0 e1d6a67b021e 0c192fcfd65b 2024-03-21T12:44:00.819 INFO:teuthology.orchestra.run.smithi022.stdout:osd.7 smithi189 running (10m) 2m ago 10m 528M 2704M 17.2.0 e1d6a67b021e 3d0723e1f863 2024-03-21T12:44:00.819 INFO:teuthology.orchestra.run.smithi022.stdout:prometheus.smithi022 smithi022 *:9095 running (4m) 31s ago 14m 73.2M - 2.33.4 514e6a882f6e 8171bbe41d2c 2024-03-21T12:44:00.883 INFO:teuthology.orchestra.run.smithi022.stdout: 5 238658 10.59 MB/sec execute 420 sec latency 60.702 ms 2024-03-21T12:44:01.041 INFO:journalctl@ceph.osd.3.smithi022.stdout:Mar 21 12:44:00 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-3[45794]: 2024-03-21T12:44:00.957+0000 7f0e0fe9f700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-21T12:44:01.041 INFO:journalctl@ceph.osd.3.smithi022.stdout:Mar 21 12:44:00 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-3[45794]: 2024-03-21T12:44:00.957+0000 7f0e0fe9f700 -1 osd.3 109 *** Got signal Terminated *** 2024-03-21T12:44:01.041 INFO:journalctl@ceph.osd.3.smithi022.stdout:Mar 21 12:44:00 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-3[45794]: 2024-03-21T12:44:00.957+0000 7f0e0fe9f700 -1 osd.3 109 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-21T12:44:01.394 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-03-21T12:44:01.394 INFO:teuthology.orchestra.run.smithi022.stdout: "mon": { 2024-03-21T12:44:01.394 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 2 2024-03-21T12:44:01.394 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:44:01.395 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr": { 2024-03-21T12:44:01.395 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 2 2024-03-21T12:44:01.395 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:44:01.395 INFO:teuthology.orchestra.run.smithi022.stdout: "osd": { 2024-03-21T12:44:01.395 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-03-21T12:44:01.395 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 3 2024-03-21T12:44:01.395 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:44:01.395 INFO:teuthology.orchestra.run.smithi022.stdout: "mds": { 2024-03-21T12:44:01.395 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-21T12:44:01.395 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:44:01.395 INFO:teuthology.orchestra.run.smithi022.stdout: "overall": { 2024-03-21T12:44:01.395 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-03-21T12:44:01.395 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 7 2024-03-21T12:44:01.395 INFO:teuthology.orchestra.run.smithi022.stdout: } 2024-03-21T12:44:01.396 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-03-21T12:44:01.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:01 smithi022 ceph-mon[67047]: from='client.34306 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:44:01.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:01 smithi022 ceph-mon[67047]: from='client.34310 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:44:01.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:01 smithi022 ceph-mon[67047]: osd.3 marked itself down and dead 2024-03-21T12:44:01.740 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-03-21T12:44:01.741 INFO:teuthology.orchestra.run.smithi022.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93", 2024-03-21T12:44:01.741 INFO:teuthology.orchestra.run.smithi022.stdout: "in_progress": true, 2024-03-21T12:44:01.741 INFO:teuthology.orchestra.run.smithi022.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-21T12:44:01.741 INFO:teuthology.orchestra.run.smithi022.stdout: "services_complete": [ 2024-03-21T12:44:01.741 INFO:teuthology.orchestra.run.smithi022.stdout: "mon", 2024-03-21T12:44:01.741 INFO:teuthology.orchestra.run.smithi022.stdout: "crash", 2024-03-21T12:44:01.741 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr" 2024-03-21T12:44:01.741 INFO:teuthology.orchestra.run.smithi022.stdout: ], 2024-03-21T12:44:01.741 INFO:teuthology.orchestra.run.smithi022.stdout: "progress": "9/23 daemons upgraded", 2024-03-21T12:44:01.741 INFO:teuthology.orchestra.run.smithi022.stdout: "message": "Currently upgrading osd daemons", 2024-03-21T12:44:01.741 INFO:teuthology.orchestra.run.smithi022.stdout: "is_paused": false 2024-03-21T12:44:01.741 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-03-21T12:44:01.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:01 smithi189 ceph-mon[53160]: from='client.34306 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:44:01.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:01 smithi189 ceph-mon[53160]: from='client.34310 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:44:01.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:01 smithi189 ceph-mon[53160]: osd.3 marked itself down and dead 2024-03-21T12:44:01.883 INFO:teuthology.orchestra.run.smithi022.stdout: 5 238821 10.58 MB/sec execute 421 sec latency 559.359 ms 2024-03-21T12:44:02.156 INFO:teuthology.orchestra.run.smithi022.stdout:HEALTH_WARN 1 failed cephadm daemon(s); 1 osds down 2024-03-21T12:44:02.156 INFO:teuthology.orchestra.run.smithi022.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-21T12:44:02.156 INFO:teuthology.orchestra.run.smithi022.stdout: daemon grafana.smithi022 on smithi022 is in error state 2024-03-21T12:44:02.156 INFO:teuthology.orchestra.run.smithi022.stdout:[WRN] OSD_DOWN: 1 osds down 2024-03-21T12:44:02.156 INFO:teuthology.orchestra.run.smithi022.stdout: osd.3 (root=default,host=smithi022) is down 2024-03-21T12:44:02.333 INFO:journalctl@ceph.osd.3.smithi022.stdout:Mar 21 12:44:02 smithi022 podman[80339]: 2024-03-21 12:44:02.049191069 +0000 UTC m=+1.251834333 container died b1462ffc1b9e7de941cfbc496e81970bfd611ce1951f6d75d206e6697d901c96 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-3, RELEASE=HEAD, architecture=x86_64, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , version=8, com.redhat.component=centos-stream-container, vcs-type=git, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-03-21T12:44:02.588 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:02 smithi022 ceph-mon[67047]: pgmap v164: 97 pgs: 97 active+clean; 338 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.8 MiB/s wr, 514 op/s 2024-03-21T12:44:02.588 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:02 smithi022 ceph-mon[67047]: from='client.34314 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:44:02.588 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:02 smithi022 ceph-mon[67047]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-21T12:44:02.589 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:02 smithi022 ceph-mon[67047]: osdmap e110: 8 total, 7 up, 8 in 2024-03-21T12:44:02.589 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:02 smithi022 ceph-mon[67047]: from='client.? 172.21.15.22:0/2710821886' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:44:02.589 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:02 smithi022 ceph-mon[67047]: from='client.? 172.21.15.22:0/2113078209' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-21T12:44:02.589 INFO:journalctl@ceph.osd.3.smithi022.stdout:Mar 21 12:44:02 smithi022 podman[80339]: 2024-03-21 12:44:02.44928163 +0000 UTC m=+1.651924900 container cleanup b1462ffc1b9e7de941cfbc496e81970bfd611ce1951f6d75d206e6697d901c96 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-3, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, ceph=True, vcs-type=git, name=centos-stream, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, architecture=x86_64, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, distribution-scope=public) 2024-03-21T12:44:02.589 INFO:journalctl@ceph.osd.3.smithi022.stdout:Mar 21 12:44:02 smithi022 bash[80339]: ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-3 2024-03-21T12:44:02.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:02 smithi189 ceph-mon[53160]: pgmap v164: 97 pgs: 97 active+clean; 338 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.8 MiB/s wr, 514 op/s 2024-03-21T12:44:02.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:02 smithi189 ceph-mon[53160]: from='client.34314 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:44:02.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:02 smithi189 ceph-mon[53160]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-21T12:44:02.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:02 smithi189 ceph-mon[53160]: osdmap e110: 8 total, 7 up, 8 in 2024-03-21T12:44:02.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:02 smithi189 ceph-mon[53160]: from='client.? 172.21.15.22:0/2710821886' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:44:02.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:02 smithi189 ceph-mon[53160]: from='client.? 172.21.15.22:0/2113078209' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-21T12:44:02.851 INFO:journalctl@ceph.osd.3.smithi022.stdout:Mar 21 12:44:02 smithi022 podman[80423]: 2024-03-21 12:44:02.776276622 +0000 UTC m=+0.729129921 container remove b1462ffc1b9e7de941cfbc496e81970bfd611ce1951f6d75d206e6697d901c96 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-3, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, name=centos-stream, io.k8s.display-name=CentOS Stream 8, release=754, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, io.openshift.expose-services=, version=8) 2024-03-21T12:44:02.883 INFO:teuthology.orchestra.run.smithi022.stdout: 5 238976 10.57 MB/sec execute 422 sec latency 1073.931 ms 2024-03-21T12:44:03.360 INFO:journalctl@ceph.osd.3.smithi022.stdout:Mar 21 12:44:03 smithi022 podman[80514]: 2024-03-21 12:44:03.242846741 +0000 UTC m=+0.033739134 image pull 19e12293512ff7978c8adb5f72a829060dd47f07bfd6903b6b3178d2ebc29e4e quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93 2024-03-21T12:44:03.360 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:03 smithi022 ceph-mon[67047]: from='client.34322 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:44:03.361 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:03 smithi022 ceph-mon[67047]: osdmap e111: 8 total, 7 up, 8 in 2024-03-21T12:44:03.611 INFO:journalctl@ceph.osd.3.smithi022.stdout:Mar 21 12:44:03 smithi022 podman[80514]: 2024-03-21 12:44:03.600577338 +0000 UTC m=+0.391469723 container create 56f307889714052c4716ed76d36d4cceee14c0ece03c78ed14cc10eae9bb4c5c (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-3-deactivate, io.buildah.version=1.33.5, ceph=True, org.label-schema.build-date=20240319, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30) 2024-03-21T12:44:03.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:03 smithi189 ceph-mon[53160]: from='client.34322 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:44:03.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:03 smithi189 ceph-mon[53160]: osdmap e111: 8 total, 7 up, 8 in 2024-03-21T12:44:03.883 INFO:teuthology.orchestra.run.smithi022.stdout: 5 239376 10.57 MB/sec execute 423 sec latency 35.818 ms 2024-03-21T12:44:04.116 INFO:journalctl@ceph.osd.3.smithi022.stdout:Mar 21 12:44:04 smithi022 podman[80514]: 2024-03-21 12:44:04.016065271 +0000 UTC m=+0.806957657 container init 56f307889714052c4716ed76d36d4cceee14c0ece03c78ed14cc10eae9bb4c5c (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-3-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240319, io.buildah.version=1.33.5, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30) 2024-03-21T12:44:04.116 INFO:journalctl@ceph.osd.3.smithi022.stdout:Mar 21 12:44:04 smithi022 podman[80514]: 2024-03-21 12:44:04.025439372 +0000 UTC m=+0.816331748 container start 56f307889714052c4716ed76d36d4cceee14c0ece03c78ed14cc10eae9bb4c5c (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-3-deactivate, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240319, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-21T12:44:04.370 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:04 smithi022 ceph-mon[67047]: pgmap v167: 97 pgs: 24 peering, 73 active+clean; 342 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 5.8 MiB/s wr, 570 op/s 2024-03-21T12:44:04.370 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:04 smithi022 ceph-mon[67047]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-03-21T12:44:04.371 INFO:journalctl@ceph.osd.3.smithi022.stdout:Mar 21 12:44:04 smithi022 podman[80514]: 2024-03-21 12:44:04.202256203 +0000 UTC m=+0.993148584 container attach 56f307889714052c4716ed76d36d4cceee14c0ece03c78ed14cc10eae9bb4c5c (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-3-deactivate, ceph=True, org.label-schema.schema-version=1.0, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, GIT_BRANCH=HEAD, org.label-schema.build-date=20240319, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True) 2024-03-21T12:44:04.621 INFO:journalctl@ceph.osd.3.smithi022.stdout:Mar 21 12:44:04 smithi022 podman[80514]: 2024-03-21 12:44:04.477755047 +0000 UTC m=+1.268647423 container died 56f307889714052c4716ed76d36d4cceee14c0ece03c78ed14cc10eae9bb4c5c (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-3-deactivate, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240319, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True) 2024-03-21T12:44:04.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:04 smithi189 ceph-mon[53160]: pgmap v167: 97 pgs: 24 peering, 73 active+clean; 342 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 5.8 MiB/s wr, 570 op/s 2024-03-21T12:44:04.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:04 smithi189 ceph-mon[53160]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-03-21T12:44:04.883 INFO:teuthology.orchestra.run.smithi022.stdout: 5 239728 10.55 MB/sec execute 424 sec latency 35.686 ms 2024-03-21T12:44:05.383 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:05 smithi022 ceph-mon[67047]: pgmap v168: 97 pgs: 4 active+undersized, 6 active+undersized+degraded, 24 peering, 63 active+clean; 338 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.9 MiB/s wr, 600 op/s; 30/1659 objects degraded (1.808%) 2024-03-21T12:44:05.639 INFO:journalctl@ceph.osd.3.smithi022.stdout:Mar 21 12:44:05 smithi022 podman[80540]: 2024-03-21 12:44:05.408977733 +0000 UTC m=+0.924867556 container remove 56f307889714052c4716ed76d36d4cceee14c0ece03c78ed14cc10eae9bb4c5c (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-3-deactivate, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.build-date=20240319, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-03-21T12:44:05.639 INFO:journalctl@ceph.osd.3.smithi022.stdout:Mar 21 12:44:05 smithi022 systemd[1]: ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715@osd.3.service: Deactivated successfully. 2024-03-21T12:44:05.639 INFO:journalctl@ceph.osd.3.smithi022.stdout:Mar 21 12:44:05 smithi022 systemd[1]: Stopped Ceph osd.3 for 3fc7f0f2-e77e-11ee-95cd-87774f69a715. 2024-03-21T12:44:05.639 INFO:journalctl@ceph.osd.3.smithi022.stdout:Mar 21 12:44:05 smithi022 systemd[1]: ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715@osd.3.service: Consumed 4min 50.411s CPU time. 2024-03-21T12:44:05.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:05 smithi189 ceph-mon[53160]: pgmap v168: 97 pgs: 4 active+undersized, 6 active+undersized+degraded, 24 peering, 63 active+clean; 338 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.9 MiB/s wr, 600 op/s; 30/1659 objects degraded (1.808%) 2024-03-21T12:44:05.883 INFO:teuthology.orchestra.run.smithi022.stdout: 5 240062 10.54 MB/sec execute 425 sec latency 42.218 ms 2024-03-21T12:44:05.891 INFO:journalctl@ceph.osd.3.smithi022.stdout:Mar 21 12:44:05 smithi022 systemd[1]: Starting Ceph osd.3 for 3fc7f0f2-e77e-11ee-95cd-87774f69a715... 2024-03-21T12:44:06.146 INFO:journalctl@ceph.osd.3.smithi022.stdout:Mar 21 12:44:06 smithi022 podman[80632]: 2024-03-21 12:44:05.906173248 +0000 UTC m=+0.024861462 image pull 19e12293512ff7978c8adb5f72a829060dd47f07bfd6903b6b3178d2ebc29e4e quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93 2024-03-21T12:44:06.401 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:06 smithi022 ceph-mon[67047]: Health check failed: Degraded data redundancy: 30/1659 objects degraded (1.808%), 6 pgs degraded (PG_DEGRADED) 2024-03-21T12:44:06.401 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:06 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:44:06.401 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:06 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:44:06.401 INFO:journalctl@ceph.osd.3.smithi022.stdout:Mar 21 12:44:06 smithi022 podman[80632]: 2024-03-21 12:44:06.39358898 +0000 UTC m=+0.512277194 container create ff56617eade4ecdbb62a13e0edec380f14372da09f35d7003fdee3afd4eef068 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-3-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, maintainer=Guillaume Abrioux , ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_CLEAN=True, org.label-schema.build-date=20240319) 2024-03-21T12:44:06.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:06 smithi189 ceph-mon[53160]: Health check failed: Degraded data redundancy: 30/1659 objects degraded (1.808%), 6 pgs degraded (PG_DEGRADED) 2024-03-21T12:44:06.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:06 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:44:06.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:06 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:44:06.883 INFO:teuthology.orchestra.run.smithi022.stdout: 5 240383 10.54 MB/sec execute 426 sec latency 50.999 ms 2024-03-21T12:44:06.940 INFO:journalctl@ceph.osd.3.smithi022.stdout:Mar 21 12:44:06 smithi022 podman[80632]: 2024-03-21 12:44:06.779328338 +0000 UTC m=+0.898016551 container init ff56617eade4ecdbb62a13e0edec380f14372da09f35d7003fdee3afd4eef068 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-3-activate, org.label-schema.schema-version=1.0, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240319, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-03-21T12:44:06.940 INFO:journalctl@ceph.osd.3.smithi022.stdout:Mar 21 12:44:06 smithi022 podman[80632]: 2024-03-21 12:44:06.786909152 +0000 UTC m=+0.905597355 container start ff56617eade4ecdbb62a13e0edec380f14372da09f35d7003fdee3afd4eef068 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-3-activate, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240319, GIT_BRANCH=HEAD, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-03-21T12:44:07.203 INFO:journalctl@ceph.osd.3.smithi022.stdout:Mar 21 12:44:06 smithi022 podman[80632]: 2024-03-21 12:44:06.950342644 +0000 UTC m=+1.069030878 container attach ff56617eade4ecdbb62a13e0edec380f14372da09f35d7003fdee3afd4eef068 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-3-activate, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.build-date=20240319, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-03-21T12:44:07.476 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:07 smithi022 ceph-mon[67047]: pgmap v169: 97 pgs: 4 active+undersized, 6 active+undersized+degraded, 24 peering, 63 active+clean; 338 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 2.5 MiB/s wr, 198 op/s; 30/1659 objects degraded (1.808%) 2024-03-21T12:44:07.476 INFO:journalctl@ceph.osd.3.smithi022.stdout:Mar 21 12:44:07 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-3-activate[80645]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-03-21T12:44:07.476 INFO:journalctl@ceph.osd.3.smithi022.stdout:Mar 21 12:44:07 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-3-activate[80645]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-03-21T12:44:07.476 INFO:journalctl@ceph.osd.3.smithi022.stdout:Mar 21 12:44:07 smithi022 bash[80632]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-03-21T12:44:07.476 INFO:journalctl@ceph.osd.3.smithi022.stdout:Mar 21 12:44:07 smithi022 bash[80632]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-03-21T12:44:07.476 INFO:journalctl@ceph.osd.3.smithi022.stdout:Mar 21 12:44:07 smithi022 bash[80632]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-03-21T12:44:07.476 INFO:journalctl@ceph.osd.3.smithi022.stdout:Mar 21 12:44:07 smithi022 bash[80632]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-03-21T12:44:07.476 INFO:journalctl@ceph.osd.3.smithi022.stdout:Mar 21 12:44:07 smithi022 bash[80632]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-03-21T12:44:07.476 INFO:journalctl@ceph.osd.3.smithi022.stdout:Mar 21 12:44:07 smithi022 bash[80632]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-03-21T12:44:07.477 INFO:journalctl@ceph.osd.3.smithi022.stdout:Mar 21 12:44:07 smithi022 bash[80632]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-03-21T12:44:07.477 INFO:journalctl@ceph.osd.3.smithi022.stdout:Mar 21 12:44:07 smithi022 bash[80632]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-03-21T12:44:07.477 INFO:journalctl@ceph.osd.3.smithi022.stdout:Mar 21 12:44:07 smithi022 bash[80632]: --> ceph-volume raw activate successful for osd ID: 3 2024-03-21T12:44:07.477 INFO:journalctl@ceph.osd.3.smithi022.stdout:Mar 21 12:44:07 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-3-activate[80645]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-03-21T12:44:07.477 INFO:journalctl@ceph.osd.3.smithi022.stdout:Mar 21 12:44:07 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-3-activate[80645]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-03-21T12:44:07.477 INFO:journalctl@ceph.osd.3.smithi022.stdout:Mar 21 12:44:07 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-3-activate[80645]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-03-21T12:44:07.477 INFO:journalctl@ceph.osd.3.smithi022.stdout:Mar 21 12:44:07 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-3-activate[80645]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-03-21T12:44:07.477 INFO:journalctl@ceph.osd.3.smithi022.stdout:Mar 21 12:44:07 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-3-activate[80645]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-03-21T12:44:07.477 INFO:journalctl@ceph.osd.3.smithi022.stdout:Mar 21 12:44:07 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-3-activate[80645]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-03-21T12:44:07.477 INFO:journalctl@ceph.osd.3.smithi022.stdout:Mar 21 12:44:07 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-3-activate[80645]: --> ceph-volume raw activate successful for osd ID: 3 2024-03-21T12:44:07.477 INFO:journalctl@ceph.osd.3.smithi022.stdout:Mar 21 12:44:07 smithi022 podman[80632]: 2024-03-21 12:44:07.354715241 +0000 UTC m=+1.473403454 container died ff56617eade4ecdbb62a13e0edec380f14372da09f35d7003fdee3afd4eef068 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-3-activate, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240319, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe) 2024-03-21T12:44:07.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:07 smithi189 ceph-mon[53160]: pgmap v169: 97 pgs: 4 active+undersized, 6 active+undersized+degraded, 24 peering, 63 active+clean; 338 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 2.5 MiB/s wr, 198 op/s; 30/1659 objects degraded (1.808%) 2024-03-21T12:44:07.883 INFO:teuthology.orchestra.run.smithi022.stdout: 5 240592 10.53 MB/sec execute 427 sec latency 82.314 ms 2024-03-21T12:44:08.766 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:08 smithi022 ceph-mon[67047]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-03-21T12:44:08.766 INFO:journalctl@ceph.osd.3.smithi022.stdout:Mar 21 12:44:08 smithi022 podman[80789]: 2024-03-21 12:44:08.534868418 +0000 UTC m=+1.175265249 container remove ff56617eade4ecdbb62a13e0edec380f14372da09f35d7003fdee3afd4eef068 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-3-activate, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240319, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.33.5) 2024-03-21T12:44:08.883 INFO:teuthology.orchestra.run.smithi022.stdout: 5 240880 10.53 MB/sec execute 428 sec latency 95.571 ms 2024-03-21T12:44:08.910 INFO:journalctl@ceph.osd.3.smithi022.stdout:Mar 21 12:44:08 smithi022 podman[80847]: 2024-03-21 12:44:08.707720871 +0000 UTC m=+0.020580927 image pull 19e12293512ff7978c8adb5f72a829060dd47f07bfd6903b6b3178d2ebc29e4e quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93 2024-03-21T12:44:08.910 INFO:journalctl@ceph.osd.3.smithi022.stdout:Mar 21 12:44:08 smithi022 podman[80847]: 2024-03-21 12:44:08.894912044 +0000 UTC m=+0.207772123 container create 61a63b67d9aff27721d31354e6c7933708b0365d59e532f2269070bd8a8c1412 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-3, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240319, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-03-21T12:44:09.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:08 smithi189 ceph-mon[53160]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-03-21T12:44:09.271 INFO:journalctl@ceph.osd.3.smithi022.stdout:Mar 21 12:44:09 smithi022 podman[80847]: 2024-03-21 12:44:09.256957425 +0000 UTC m=+0.569817470 container init 61a63b67d9aff27721d31354e6c7933708b0365d59e532f2269070bd8a8c1412 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-3, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240319, org.label-schema.vendor=CentOS, ceph=True) 2024-03-21T12:44:09.271 INFO:journalctl@ceph.osd.3.smithi022.stdout:Mar 21 12:44:09 smithi022 podman[80847]: 2024-03-21 12:44:09.26194782 +0000 UTC m=+0.574807857 container start 61a63b67d9aff27721d31354e6c7933708b0365d59e532f2269070bd8a8c1412 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-3, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240319, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-03-21T12:44:09.535 INFO:journalctl@ceph.osd.3.smithi022.stdout:Mar 21 12:44:09 smithi022 ceph-osd[80865]: -- 172.21.15.22:0/4063872077 <== mon.0 v2:172.21.15.22:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5618efa49680 con 0x5618efa13000 2024-03-21T12:44:09.535 INFO:journalctl@ceph.osd.3.smithi022.stdout:Mar 21 12:44:09 smithi022 bash[80847]: 61a63b67d9aff27721d31354e6c7933708b0365d59e532f2269070bd8a8c1412 2024-03-21T12:44:09.535 INFO:journalctl@ceph.osd.3.smithi022.stdout:Mar 21 12:44:09 smithi022 systemd[1]: Started Ceph osd.3 for 3fc7f0f2-e77e-11ee-95cd-87774f69a715. 2024-03-21T12:44:09.804 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:09 smithi022 ceph-mon[67047]: pgmap v170: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 337 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.5 MiB/s wr, 493 op/s; 214/1689 objects degraded (12.670%) 2024-03-21T12:44:09.805 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:09 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:44:09.805 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:09 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:44:09.805 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:09 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:44:09.883 INFO:teuthology.orchestra.run.smithi022.stdout: 5 241138 10.52 MB/sec execute 429 sec latency 76.116 ms 2024-03-21T12:44:10.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:09 smithi189 ceph-mon[53160]: pgmap v170: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 337 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.5 MiB/s wr, 493 op/s; 214/1689 objects degraded (12.670%) 2024-03-21T12:44:10.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:09 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:44:10.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:09 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:44:10.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:09 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:44:10.056 INFO:journalctl@ceph.osd.3.smithi022.stdout:Mar 21 12:44:09 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-3[80861]: 2024-03-21T12:44:09.833+0000 7fc2c1d7c7c0 -1 Falling back to public interface 2024-03-21T12:44:10.883 INFO:teuthology.orchestra.run.smithi022.stdout: 5 241382 10.50 MB/sec execute 430 sec latency 46.758 ms 2024-03-21T12:44:11.834 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:11 smithi022 ceph-mon[67047]: pgmap v171: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 337 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.8 MiB/s wr, 425 op/s; 214/1689 objects degraded (12.670%) 2024-03-21T12:44:11.883 INFO:teuthology.orchestra.run.smithi022.stdout: 5 241689 10.50 MB/sec execute 431 sec latency 80.772 ms 2024-03-21T12:44:12.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:11 smithi189 ceph-mon[53160]: pgmap v171: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 337 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.8 MiB/s wr, 425 op/s; 214/1689 objects degraded (12.670%) 2024-03-21T12:44:12.849 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:12 smithi022 ceph-mon[67047]: Health check update: Degraded data redundancy: 215/1632 objects degraded (13.174%), 27 pgs degraded (PG_DEGRADED) 2024-03-21T12:44:12.885 INFO:teuthology.orchestra.run.smithi022.stdout: 5 242092 10.51 MB/sec execute 432 sec latency 36.957 ms 2024-03-21T12:44:13.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:12 smithi189 ceph-mon[53160]: Health check update: Degraded data redundancy: 215/1632 objects degraded (13.174%), 27 pgs degraded (PG_DEGRADED) 2024-03-21T12:44:13.866 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:13 smithi022 ceph-mon[67047]: pgmap v172: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 334 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.0 MiB/s wr, 409 op/s; 215/1632 objects degraded (13.174%) 2024-03-21T12:44:13.885 INFO:teuthology.orchestra.run.smithi022.stdout: 5 242430 10.51 MB/sec execute 433 sec latency 55.141 ms 2024-03-21T12:44:14.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:13 smithi189 ceph-mon[53160]: pgmap v172: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 334 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.0 MiB/s wr, 409 op/s; 215/1632 objects degraded (13.174%) 2024-03-21T12:44:14.886 INFO:teuthology.orchestra.run.smithi022.stdout: 5 242747 10.51 MB/sec execute 434 sec latency 67.855 ms 2024-03-21T12:44:15.675 INFO:journalctl@ceph.osd.3.smithi022.stdout:Mar 21 12:44:15 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-3[80861]: 2024-03-21T12:44:15.631+0000 7fc2c1d7c7c0 -1 osd.3 0 read_superblock omap replica is missing. 2024-03-21T12:44:15.886 INFO:teuthology.orchestra.run.smithi022.stdout: 5 243072 10.50 MB/sec execute 435 sec latency 45.450 ms 2024-03-21T12:44:15.930 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:15 smithi022 ceph-mon[67047]: pgmap v173: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 339 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.9 MiB/s wr, 422 op/s; 225/1656 objects degraded (13.587%) 2024-03-21T12:44:16.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:15 smithi189 ceph-mon[53160]: pgmap v173: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 339 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.9 MiB/s wr, 422 op/s; 225/1656 objects degraded (13.587%) 2024-03-21T12:44:16.886 INFO:teuthology.orchestra.run.smithi022.stdout: 5 243416 10.49 MB/sec execute 436 sec latency 37.242 ms 2024-03-21T12:44:17.451 INFO:journalctl@ceph.osd.3.smithi022.stdout:Mar 21 12:44:17 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-3[80861]: 2024-03-21T12:44:17.227+0000 7fc2c1d7c7c0 -1 osd.3 109 log_to_monitors true 2024-03-21T12:44:17.886 INFO:teuthology.orchestra.run.smithi022.stdout: 5 243708 10.48 MB/sec execute 437 sec latency 46.349 ms 2024-03-21T12:44:17.956 INFO:journalctl@ceph.osd.3.smithi022.stdout:Mar 21 12:44:17 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-3[80861]: 2024-03-21T12:44:17.828+0000 7fc2b31d4700 -1 osd.3 109 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-21T12:44:17.956 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:17 smithi022 ceph-mon[67047]: pgmap v174: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 339 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.8 MiB/s wr, 402 op/s; 225/1656 objects degraded (13.587%) 2024-03-21T12:44:17.956 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:17 smithi022 ceph-mon[67047]: from='osd.3 [v2:172.21.15.22:6826/1791361959,v1:172.21.15.22:6827/1791361959]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-03-21T12:44:17.956 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:17 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:44:17.956 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:17 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:44:18.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:17 smithi189 ceph-mon[53160]: pgmap v174: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 339 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.8 MiB/s wr, 402 op/s; 225/1656 objects degraded (13.587%) 2024-03-21T12:44:18.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:17 smithi189 ceph-mon[53160]: from='osd.3 [v2:172.21.15.22:6826/1791361959,v1:172.21.15.22:6827/1791361959]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-03-21T12:44:18.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:17 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:44:18.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:17 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:44:18.886 INFO:teuthology.orchestra.run.smithi022.stdout: 5 244053 10.49 MB/sec execute 438 sec latency 95.565 ms 2024-03-21T12:44:19.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:18 smithi022 ceph-mon[67047]: from='osd.3 [v2:172.21.15.22:6826/1791361959,v1:172.21.15.22:6827/1791361959]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-03-21T12:44:19.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:18 smithi022 ceph-mon[67047]: osdmap e112: 8 total, 7 up, 8 in 2024-03-21T12:44:19.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:18 smithi022 ceph-mon[67047]: from='osd.3 [v2:172.21.15.22:6826/1791361959,v1:172.21.15.22:6827/1791361959]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]: dispatch 2024-03-21T12:44:19.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:18 smithi022 ceph-mon[67047]: Health check update: Degraded data redundancy: 225/1656 objects degraded (13.587%), 27 pgs degraded (PG_DEGRADED) 2024-03-21T12:44:19.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:18 smithi189 ceph-mon[53160]: from='osd.3 [v2:172.21.15.22:6826/1791361959,v1:172.21.15.22:6827/1791361959]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-03-21T12:44:19.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:18 smithi189 ceph-mon[53160]: osdmap e112: 8 total, 7 up, 8 in 2024-03-21T12:44:19.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:18 smithi189 ceph-mon[53160]: from='osd.3 [v2:172.21.15.22:6826/1791361959,v1:172.21.15.22:6827/1791361959]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]: dispatch 2024-03-21T12:44:19.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:18 smithi189 ceph-mon[53160]: Health check update: Degraded data redundancy: 225/1656 objects degraded (13.587%), 27 pgs degraded (PG_DEGRADED) 2024-03-21T12:44:19.886 INFO:teuthology.orchestra.run.smithi022.stdout: 5 244053 10.46 MB/sec execute 439 sec latency 1095.600 ms 2024-03-21T12:44:20.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:19 smithi022 ceph-mon[67047]: pgmap v176: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 338 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.3 MiB/s wr, 437 op/s; 228/1689 objects degraded (13.499%) 2024-03-21T12:44:20.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:19 smithi022 ceph-mon[67047]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-21T12:44:20.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:19 smithi022 ceph-mon[67047]: osd.3 [v2:172.21.15.22:6826/1791361959,v1:172.21.15.22:6827/1791361959] boot 2024-03-21T12:44:20.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:19 smithi022 ceph-mon[67047]: osdmap e113: 8 total, 8 up, 8 in 2024-03-21T12:44:20.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:19 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-21T12:44:20.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:19 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:44:20.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:19 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:44:20.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:19 smithi189 ceph-mon[53160]: pgmap v176: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 338 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.3 MiB/s wr, 437 op/s; 228/1689 objects degraded (13.499%) 2024-03-21T12:44:20.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:19 smithi189 ceph-mon[53160]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-21T12:44:20.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:19 smithi189 ceph-mon[53160]: osd.3 [v2:172.21.15.22:6826/1791361959,v1:172.21.15.22:6827/1791361959] boot 2024-03-21T12:44:20.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:19 smithi189 ceph-mon[53160]: osdmap e113: 8 total, 8 up, 8 in 2024-03-21T12:44:20.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:19 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-21T12:44:20.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:19 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:44:20.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:19 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:44:20.886 INFO:teuthology.orchestra.run.smithi022.stdout: 5 244053 10.44 MB/sec execute 440 sec latency 2095.649 ms 2024-03-21T12:44:21.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:20 smithi022 ceph-mon[67047]: osdmap e114: 8 total, 8 up, 8 in 2024-03-21T12:44:21.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:20 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:44:21.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:20 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:44:21.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:20 smithi022 ceph-mon[67047]: osdmap e115: 8 total, 8 up, 8 in 2024-03-21T12:44:21.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:20 smithi189 ceph-mon[53160]: osdmap e114: 8 total, 8 up, 8 in 2024-03-21T12:44:21.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:20 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:44:21.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:20 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:44:21.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:20 smithi189 ceph-mon[53160]: osdmap e115: 8 total, 8 up, 8 in 2024-03-21T12:44:21.886 INFO:teuthology.orchestra.run.smithi022.stdout: 5 244281 10.43 MB/sec execute 441 sec latency 2455.817 ms 2024-03-21T12:44:22.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:21 smithi022 ceph-mon[67047]: pgmap v179: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 338 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.6 MiB/s wr, 317 op/s; 228/1689 objects degraded (13.499%) 2024-03-21T12:44:22.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:21 smithi189 ceph-mon[53160]: pgmap v179: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 338 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.6 MiB/s wr, 317 op/s; 228/1689 objects degraded (13.499%) 2024-03-21T12:44:22.886 INFO:teuthology.orchestra.run.smithi022.stdout: 5 244297 10.40 MB/sec execute 442 sec latency 1032.859 ms 2024-03-21T12:44:23.082 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:22 smithi022 ceph-mon[67047]: osdmap e116: 8 total, 8 up, 8 in 2024-03-21T12:44:23.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:22 smithi189 ceph-mon[53160]: osdmap e116: 8 total, 8 up, 8 in 2024-03-21T12:44:23.838 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:23 smithi022 ceph-mon[67047]: pgmap v182: 97 pgs: 2 peering, 6 active+undersized, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped, 12 active+undersized+degraded, 70 active+clean; 335 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 3.1 MiB/s wr, 344 op/s; 173/1545 objects degraded (11.197%); 1004 KiB/s, 21 objects/s recovering 2024-03-21T12:44:23.838 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:23 smithi022 ceph-mon[67047]: osdmap e117: 8 total, 8 up, 8 in 2024-03-21T12:44:23.838 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:23 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:44:23.838 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:23 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:44:23.838 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:23 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:44:23.839 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:23 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:44:23.839 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:23 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:44:23.839 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:23 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:44:23.839 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:23 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:44:23.839 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:23 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:44:23.839 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:23 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:44:23.839 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:23 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:44:23.839 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:23 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-03-21T12:44:23.839 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:23 smithi022 ceph-mon[67047]: Health check update: Degraded data redundancy: 173/1545 objects degraded (11.197%), 17 pgs degraded (PG_DEGRADED) 2024-03-21T12:44:23.839 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:23 smithi022 ceph-mon[67047]: osdmap e118: 8 total, 8 up, 8 in 2024-03-21T12:44:23.886 INFO:teuthology.orchestra.run.smithi022.stdout: 5 244653 10.40 MB/sec execute 443 sec latency 109.007 ms 2024-03-21T12:44:24.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:23 smithi189 ceph-mon[53160]: pgmap v182: 97 pgs: 2 peering, 6 active+undersized, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped, 12 active+undersized+degraded, 70 active+clean; 335 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 3.1 MiB/s wr, 344 op/s; 173/1545 objects degraded (11.197%); 1004 KiB/s, 21 objects/s recovering 2024-03-21T12:44:24.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:23 smithi189 ceph-mon[53160]: osdmap e117: 8 total, 8 up, 8 in 2024-03-21T12:44:24.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:23 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:44:24.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:23 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:44:24.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:23 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:44:24.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:23 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:44:24.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:23 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:44:24.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:23 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:44:24.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:23 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:44:24.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:23 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:44:24.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:23 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:44:24.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:23 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:44:24.287 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:23 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-03-21T12:44:24.287 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:23 smithi189 ceph-mon[53160]: Health check update: Degraded data redundancy: 173/1545 objects degraded (11.197%), 17 pgs degraded (PG_DEGRADED) 2024-03-21T12:44:24.287 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:23 smithi189 ceph-mon[53160]: osdmap e118: 8 total, 8 up, 8 in 2024-03-21T12:44:24.870 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:24 smithi022 ceph-mon[67047]: pgmap v184: 97 pgs: 4 peering, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 333 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.3 KiB/s rd, 3.4 MiB/s wr, 382 op/s; 108/1509 objects degraded (7.157%); 2.9 MiB/s, 41 objects/s recovering 2024-03-21T12:44:24.870 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:24 smithi022 ceph-mon[67047]: pgmap v185: 97 pgs: 4 peering, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 333 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 4.3 MiB/s wr, 488 op/s; 108/1509 objects degraded (7.157%); 3.7 MiB/s, 53 objects/s recovering 2024-03-21T12:44:24.870 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:24 smithi022 ceph-mon[67047]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-03-21T12:44:24.870 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:24 smithi022 ceph-mon[67047]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-03-21T12:44:24.870 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:24 smithi022 ceph-mon[67047]: osdmap e119: 8 total, 8 up, 8 in 2024-03-21T12:44:24.886 INFO:teuthology.orchestra.run.smithi022.stdout: 5 244846 10.38 MB/sec execute 444 sec latency 1004.936 ms 2024-03-21T12:44:25.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:24 smithi189 ceph-mon[53160]: pgmap v184: 97 pgs: 4 peering, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 333 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.3 KiB/s rd, 3.4 MiB/s wr, 382 op/s; 108/1509 objects degraded (7.157%); 2.9 MiB/s, 41 objects/s recovering 2024-03-21T12:44:25.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:24 smithi189 ceph-mon[53160]: pgmap v185: 97 pgs: 4 peering, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 333 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 4.3 MiB/s wr, 488 op/s; 108/1509 objects degraded (7.157%); 3.7 MiB/s, 53 objects/s recovering 2024-03-21T12:44:25.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:24 smithi189 ceph-mon[53160]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-03-21T12:44:25.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:24 smithi189 ceph-mon[53160]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-03-21T12:44:25.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:24 smithi189 ceph-mon[53160]: osdmap e119: 8 total, 8 up, 8 in 2024-03-21T12:44:25.886 INFO:teuthology.orchestra.run.smithi022.stdout: 5 245204 10.38 MB/sec execute 445 sec latency 33.185 ms 2024-03-21T12:44:26.886 INFO:teuthology.orchestra.run.smithi022.stdout: 5 245518 10.38 MB/sec execute 446 sec latency 92.184 ms 2024-03-21T12:44:26.908 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:26 smithi022 ceph-mon[67047]: pgmap v188: 97 pgs: 4 peering, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 333 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 104 KiB/s wr, 22 op/s; 108/1509 objects degraded (7.157%); 2.3 MiB/s, 23 objects/s recovering 2024-03-21T12:44:27.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:26 smithi189 ceph-mon[53160]: pgmap v188: 97 pgs: 4 peering, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 333 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 104 KiB/s wr, 22 op/s; 108/1509 objects degraded (7.157%); 2.3 MiB/s, 23 objects/s recovering 2024-03-21T12:44:27.886 INFO:teuthology.orchestra.run.smithi022.stdout: 5 245873 10.38 MB/sec execute 447 sec latency 46.356 ms 2024-03-21T12:44:28.886 INFO:teuthology.orchestra.run.smithi022.stdout: 5 246150 10.38 MB/sec execute 448 sec latency 46.454 ms 2024-03-21T12:44:28.939 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:28 smithi022 ceph-mon[67047]: pgmap v189: 97 pgs: 2 peering, 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 335 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 3.8 KiB/s rd, 3.3 MiB/s wr, 364 op/s; 64/1620 objects degraded (3.951%); 1.9 MiB/s, 19 objects/s recovering 2024-03-21T12:44:28.940 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:28 smithi022 ceph-mon[67047]: Health check update: Degraded data redundancy: 64/1620 objects degraded (3.951%), 2 pgs degraded (PG_DEGRADED) 2024-03-21T12:44:29.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:28 smithi189 ceph-mon[53160]: pgmap v189: 97 pgs: 2 peering, 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 335 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 3.8 KiB/s rd, 3.3 MiB/s wr, 364 op/s; 64/1620 objects degraded (3.951%); 1.9 MiB/s, 19 objects/s recovering 2024-03-21T12:44:29.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:28 smithi189 ceph-mon[53160]: Health check update: Degraded data redundancy: 64/1620 objects degraded (3.951%), 2 pgs degraded (PG_DEGRADED) 2024-03-21T12:44:29.886 INFO:teuthology.orchestra.run.smithi022.stdout: 5 246372 10.37 MB/sec execute 449 sec latency 54.115 ms 2024-03-21T12:44:29.969 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:29 smithi022 ceph-mon[67047]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 64/1620 objects degraded (3.951%), 2 pgs degraded) 2024-03-21T12:44:30.186 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:29 smithi189 ceph-mon[53160]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 64/1620 objects degraded (3.951%), 2 pgs degraded) 2024-03-21T12:44:30.886 INFO:teuthology.orchestra.run.smithi022.stdout: 5 246650 10.37 MB/sec execute 450 sec latency 69.911 ms 2024-03-21T12:44:31.017 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:30 smithi022 ceph-mon[67047]: pgmap v190: 97 pgs: 97 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 3.3 KiB/s rd, 3.2 MiB/s wr, 314 op/s; 548 KiB/s, 23 objects/s recovering 2024-03-21T12:44:31.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:30 smithi189 ceph-mon[53160]: pgmap v190: 97 pgs: 97 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 3.3 KiB/s rd, 3.2 MiB/s wr, 314 op/s; 548 KiB/s, 23 objects/s recovering 2024-03-21T12:44:31.886 INFO:teuthology.orchestra.run.smithi022.stdout: 5 246926 10.36 MB/sec execute 451 sec latency 39.386 ms 2024-03-21T12:44:32.616 INFO:teuthology.orchestra.run.smithi022.stdout:true 2024-03-21T12:44:32.886 INFO:teuthology.orchestra.run.smithi022.stdout: 5 247204 10.35 MB/sec execute 452 sec latency 58.290 ms 2024-03-21T12:44:32.925 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:32 smithi022 ceph-mon[67047]: pgmap v191: 97 pgs: 97 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 2.5 KiB/s rd, 2.4 MiB/s wr, 236 op/s; 411 KiB/s, 17 objects/s recovering 2024-03-21T12:44:33.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:32 smithi189 ceph-mon[53160]: pgmap v191: 97 pgs: 97 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 2.5 KiB/s rd, 2.4 MiB/s wr, 236 op/s; 411 KiB/s, 17 objects/s recovering 2024-03-21T12:44:33.441 INFO:teuthology.orchestra.run.smithi022.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-21T12:44:33.441 INFO:teuthology.orchestra.run.smithi022.stdout:alertmanager.smithi022 smithi022 *:9093,9094 running (7m) 16s ago 15m 24.1M - 0.23.0 ba2b418f427c 14eb2a3ca318 2024-03-21T12:44:33.441 INFO:teuthology.orchestra.run.smithi022.stdout:crash.smithi022 smithi022 running (3m) 16s ago 15m 6660k - 19.0.0-2267-g960fbbe4 19e12293512f 356b2ddb609a 2024-03-21T12:44:33.441 INFO:teuthology.orchestra.run.smithi022.stdout:crash.smithi189 smithi189 running (3m) 3m ago 15m 6660k - 19.0.0-2267-g960fbbe4 19e12293512f 90ee837f0500 2024-03-21T12:44:33.441 INFO:teuthology.orchestra.run.smithi022.stdout:grafana.smithi022 smithi022 *:3000 error 16s ago 15m - - 2024-03-21T12:44:33.441 INFO:teuthology.orchestra.run.smithi022.stdout:mds.foofs.smithi022.bcnxho smithi022 running (10m) 16s ago 10m 38.3M - 17.2.0 e1d6a67b021e db26b67b8ee7 2024-03-21T12:44:33.441 INFO:teuthology.orchestra.run.smithi022.stdout:mds.foofs.smithi189.lluvko smithi189 running (10m) 3m ago 10m 16.9M - 17.2.0 e1d6a67b021e cf153aff1387 2024-03-21T12:44:33.441 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.smithi022.fektzs smithi022 *:8443,9283,8765 running (5m) 16s ago 17m 506M - 19.0.0-2267-g960fbbe4 19e12293512f d57cebd306b3 2024-03-21T12:44:33.441 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.smithi189.cnxzvu smithi189 *:8443,9283,8765 running (5m) 3m ago 15m 450M - 19.0.0-2267-g960fbbe4 19e12293512f 41663955c56f 2024-03-21T12:44:33.441 INFO:teuthology.orchestra.run.smithi022.stdout:mon.smithi022 smithi022 running (4m) 16s ago 17m 50.6M 2048M 19.0.0-2267-g960fbbe4 19e12293512f c620c9071b2c 2024-03-21T12:44:33.442 INFO:teuthology.orchestra.run.smithi022.stdout:mon.smithi189 smithi189 running (4m) 3m ago 15m 35.8M 2048M 19.0.0-2267-g960fbbe4 19e12293512f b353af3d8f3b 2024-03-21T12:44:33.442 INFO:teuthology.orchestra.run.smithi022.stdout:nfs.foo.0.0.smithi022.ynhbbq smithi022 *:2049 running (9m) 16s ago 9m 277M - 4.0 e1d6a67b021e 33e48e64c60b 2024-03-21T12:44:33.442 INFO:teuthology.orchestra.run.smithi022.stdout:nfs.foo.1.0.smithi189.yzsqml smithi189 *:2049 running (9m) 3m ago 9m 85.4M - 4.0 e1d6a67b021e 48fab2cbdde2 2024-03-21T12:44:33.442 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.smithi022 smithi022 *:9100 running (6m) 16s ago 15m 18.2M - 1.3.1 1dbe0e931976 33287f803f4f 2024-03-21T12:44:33.442 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.smithi189 smithi189 *:9100 running (6m) 3m ago 14m 17.5M - 1.3.1 1dbe0e931976 b51d872e438a 2024-03-21T12:44:33.442 INFO:teuthology.orchestra.run.smithi022.stdout:osd.0 smithi022 running (2m) 16s ago 14m 336M 1937M 19.0.0-2267-g960fbbe4 19e12293512f 23a85645eeea 2024-03-21T12:44:33.442 INFO:teuthology.orchestra.run.smithi022.stdout:osd.1 smithi022 running (108s) 16s ago 14m 286M 1937M 19.0.0-2267-g960fbbe4 19e12293512f 0f90e70bbf0a 2024-03-21T12:44:33.442 INFO:teuthology.orchestra.run.smithi022.stdout:osd.2 smithi022 running (74s) 16s ago 13m 214M 1937M 19.0.0-2267-g960fbbe4 19e12293512f 78a229f593f8 2024-03-21T12:44:33.442 INFO:teuthology.orchestra.run.smithi022.stdout:osd.3 smithi022 running (24s) 16s ago 13m 12.5M 1937M 19.0.0-2267-g960fbbe4 19e12293512f 61a63b67d9af 2024-03-21T12:44:33.442 INFO:teuthology.orchestra.run.smithi022.stdout:osd.4 smithi189 running (12m) 3m ago 12m 560M 2704M 17.2.0 e1d6a67b021e 1bedc4eb02ed 2024-03-21T12:44:33.442 INFO:teuthology.orchestra.run.smithi022.stdout:osd.5 smithi189 running (12m) 3m ago 12m 407M 2704M 17.2.0 e1d6a67b021e 25fbab9786a5 2024-03-21T12:44:33.442 INFO:teuthology.orchestra.run.smithi022.stdout:osd.6 smithi189 running (11m) 3m ago 11m 578M 2704M 17.2.0 e1d6a67b021e 0c192fcfd65b 2024-03-21T12:44:33.442 INFO:teuthology.orchestra.run.smithi022.stdout:osd.7 smithi189 running (11m) 3m ago 11m 528M 2704M 17.2.0 e1d6a67b021e 3d0723e1f863 2024-03-21T12:44:33.442 INFO:teuthology.orchestra.run.smithi022.stdout:prometheus.smithi022 smithi022 *:9095 running (5m) 16s ago 14m 73.9M - 2.33.4 514e6a882f6e 8171bbe41d2c 2024-03-21T12:44:33.886 INFO:teuthology.orchestra.run.smithi022.stdout: 5 247463 10.34 MB/sec execute 453 sec latency 63.890 ms 2024-03-21T12:44:33.945 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:33 smithi022 ceph-mon[67047]: from='client.34334 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:44:33.945 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:33 smithi022 ceph-mon[67047]: from='client.34338 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:44:33.955 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-03-21T12:44:33.955 INFO:teuthology.orchestra.run.smithi022.stdout: "mon": { 2024-03-21T12:44:33.955 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 2 2024-03-21T12:44:33.955 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:44:33.955 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr": { 2024-03-21T12:44:33.955 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 2 2024-03-21T12:44:33.955 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:44:33.955 INFO:teuthology.orchestra.run.smithi022.stdout: "osd": { 2024-03-21T12:44:33.955 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-03-21T12:44:33.955 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 4 2024-03-21T12:44:33.955 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:44:33.955 INFO:teuthology.orchestra.run.smithi022.stdout: "mds": { 2024-03-21T12:44:33.955 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-21T12:44:33.956 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:44:33.956 INFO:teuthology.orchestra.run.smithi022.stdout: "overall": { 2024-03-21T12:44:33.956 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-03-21T12:44:33.956 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 8 2024-03-21T12:44:33.956 INFO:teuthology.orchestra.run.smithi022.stdout: } 2024-03-21T12:44:33.956 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-03-21T12:44:34.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:33 smithi189 ceph-mon[53160]: from='client.34334 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:44:34.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:33 smithi189 ceph-mon[53160]: from='client.34338 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:44:34.365 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-03-21T12:44:34.365 INFO:teuthology.orchestra.run.smithi022.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93", 2024-03-21T12:44:34.365 INFO:teuthology.orchestra.run.smithi022.stdout: "in_progress": true, 2024-03-21T12:44:34.366 INFO:teuthology.orchestra.run.smithi022.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-21T12:44:34.366 INFO:teuthology.orchestra.run.smithi022.stdout: "services_complete": [ 2024-03-21T12:44:34.366 INFO:teuthology.orchestra.run.smithi022.stdout: "mon", 2024-03-21T12:44:34.366 INFO:teuthology.orchestra.run.smithi022.stdout: "crash", 2024-03-21T12:44:34.366 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr" 2024-03-21T12:44:34.366 INFO:teuthology.orchestra.run.smithi022.stdout: ], 2024-03-21T12:44:34.366 INFO:teuthology.orchestra.run.smithi022.stdout: "progress": "10/23 daemons upgraded", 2024-03-21T12:44:34.366 INFO:teuthology.orchestra.run.smithi022.stdout: "message": "Currently upgrading osd daemons", 2024-03-21T12:44:34.366 INFO:teuthology.orchestra.run.smithi022.stdout: "is_paused": false 2024-03-21T12:44:34.366 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-03-21T12:44:34.886 INFO:teuthology.orchestra.run.smithi022.stdout: 5 247785 10.34 MB/sec execute 454 sec latency 59.102 ms 2024-03-21T12:44:34.896 INFO:teuthology.orchestra.run.smithi022.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-03-21T12:44:34.896 INFO:teuthology.orchestra.run.smithi022.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-21T12:44:34.896 INFO:teuthology.orchestra.run.smithi022.stdout: daemon grafana.smithi022 on smithi022 is in error state 2024-03-21T12:44:34.966 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:34 smithi022 ceph-mon[67047]: from='client.34342 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:44:34.967 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:34 smithi022 ceph-mon[67047]: pgmap v192: 97 pgs: 97 active+clean; 335 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 4.2 MiB/s wr, 394 op/s; 335 KiB/s, 14 objects/s recovering 2024-03-21T12:44:34.967 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:34 smithi022 ceph-mon[67047]: from='client.? 172.21.15.22:0/4013762153' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:44:34.967 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:34 smithi022 ceph-mon[67047]: from='client.? 172.21.15.22:0/4205702709' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-21T12:44:35.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:34 smithi189 ceph-mon[53160]: from='client.34342 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:44:35.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:34 smithi189 ceph-mon[53160]: pgmap v192: 97 pgs: 97 active+clean; 335 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 4.2 MiB/s wr, 394 op/s; 335 KiB/s, 14 objects/s recovering 2024-03-21T12:44:35.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:34 smithi189 ceph-mon[53160]: from='client.? 172.21.15.22:0/4013762153' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:44:35.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:34 smithi189 ceph-mon[53160]: from='client.? 172.21.15.22:0/4205702709' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-21T12:44:35.886 INFO:teuthology.orchestra.run.smithi022.stdout: 5 248155 10.33 MB/sec execute 455 sec latency 57.568 ms 2024-03-21T12:44:35.987 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:35 smithi022 ceph-mon[67047]: from='client.34350 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:44:35.987 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:35 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:44:35.988 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:35 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:44:35.988 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:35 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:44:36.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:35 smithi189 ceph-mon[53160]: from='client.34350 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:44:36.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:35 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:44:36.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:35 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:44:36.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:35 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:44:36.886 INFO:teuthology.orchestra.run.smithi022.stdout: 5 248497 10.33 MB/sec execute 456 sec latency 52.386 ms 2024-03-21T12:44:37.020 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:36 smithi022 ceph-mon[67047]: pgmap v193: 97 pgs: 97 active+clean; 335 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.3 KiB/s rd, 3.8 MiB/s wr, 358 op/s; 304 KiB/s, 12 objects/s recovering 2024-03-21T12:44:37.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:36 smithi189 ceph-mon[53160]: pgmap v193: 97 pgs: 97 active+clean; 335 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.3 KiB/s rd, 3.8 MiB/s wr, 358 op/s; 304 KiB/s, 12 objects/s recovering 2024-03-21T12:44:37.886 INFO:teuthology.orchestra.run.smithi022.stdout: 5 248816 10.33 MB/sec execute 457 sec latency 60.969 ms 2024-03-21T12:44:38.887 INFO:teuthology.orchestra.run.smithi022.stdout: 5 249176 10.32 MB/sec execute 458 sec latency 50.469 ms 2024-03-21T12:44:38.926 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:38 smithi022 ceph-mon[67047]: pgmap v194: 97 pgs: 97 active+clean; 333 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.9 MiB/s wr, 504 op/s; 274 KiB/s, 11 objects/s recovering 2024-03-21T12:44:38.926 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:38 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-03-21T12:44:39.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:38 smithi189 ceph-mon[53160]: pgmap v194: 97 pgs: 97 active+clean; 333 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.9 MiB/s wr, 504 op/s; 274 KiB/s, 11 objects/s recovering 2024-03-21T12:44:39.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:38 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-03-21T12:44:39.886 INFO:teuthology.orchestra.run.smithi022.stdout: 5 249512 10.32 MB/sec execute 459 sec latency 51.135 ms 2024-03-21T12:44:40.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:39 smithi189 ceph-mon[53160]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-03-21T12:44:40.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:39 smithi189 ceph-mon[53160]: Upgrade: osd.4 is safe to restart 2024-03-21T12:44:40.087 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:39 smithi022 ceph-mon[67047]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-03-21T12:44:40.087 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:39 smithi022 ceph-mon[67047]: Upgrade: osd.4 is safe to restart 2024-03-21T12:44:40.886 INFO:teuthology.orchestra.run.smithi022.stdout: 5 249761 10.32 MB/sec execute 460 sec latency 51.572 ms 2024-03-21T12:44:41.111 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:40 smithi022 ceph-mon[67047]: pgmap v195: 97 pgs: 97 active+clean; 333 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.0 MiB/s wr, 380 op/s; 212 KiB/s, 5 objects/s recovering 2024-03-21T12:44:41.112 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:40 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:44:41.112 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:40 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-03-21T12:44:41.112 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:40 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:44:41.267 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:40 smithi189 ceph-mon[53160]: pgmap v195: 97 pgs: 97 active+clean; 333 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.0 MiB/s wr, 380 op/s; 212 KiB/s, 5 objects/s recovering 2024-03-21T12:44:41.267 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:40 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:44:41.267 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:40 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-03-21T12:44:41.268 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:40 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:44:41.886 INFO:teuthology.orchestra.run.smithi022.stdout: 5 250027 10.32 MB/sec execute 461 sec latency 47.322 ms 2024-03-21T12:44:42.150 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:41 smithi022 ceph-mon[67047]: Upgrade: Updating osd.4 2024-03-21T12:44:42.150 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:41 smithi022 ceph-mon[67047]: Deploying daemon osd.4 on smithi189 2024-03-21T12:44:42.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:41 smithi189 ceph-mon[53160]: Upgrade: Updating osd.4 2024-03-21T12:44:42.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:41 smithi189 ceph-mon[53160]: Deploying daemon osd.4 on smithi189 2024-03-21T12:44:42.886 INFO:teuthology.orchestra.run.smithi022.stdout: 5 250242 10.31 MB/sec execute 462 sec latency 80.509 ms 2024-03-21T12:44:42.939 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:42 smithi022 ceph-mon[67047]: pgmap v196: 97 pgs: 97 active+clean; 333 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 3.6 MiB/s wr, 364 op/s 2024-03-21T12:44:43.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:42 smithi189 ceph-mon[53160]: pgmap v196: 97 pgs: 97 active+clean; 333 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 3.6 MiB/s wr, 364 op/s 2024-03-21T12:44:43.886 INFO:teuthology.orchestra.run.smithi022.stdout: 5 250529 10.29 MB/sec execute 463 sec latency 48.583 ms 2024-03-21T12:44:44.535 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 21 12:44:44 smithi189 systemd[1]: Stopping Ceph osd.4 for 3fc7f0f2-e77e-11ee-95cd-87774f69a715... 2024-03-21T12:44:44.786 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 21 12:44:44 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-4[31246]: 2024-03-21T12:44:44.534+0000 7fa49ee16700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-21T12:44:44.786 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 21 12:44:44 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-4[31246]: 2024-03-21T12:44:44.534+0000 7fa49ee16700 -1 osd.4 119 *** Got signal Terminated *** 2024-03-21T12:44:44.786 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 21 12:44:44 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-4[31246]: 2024-03-21T12:44:44.534+0000 7fa49ee16700 -1 osd.4 119 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-21T12:44:44.886 INFO:teuthology.orchestra.run.smithi022.stdout: 5 250753 10.28 MB/sec execute 464 sec latency 64.634 ms 2024-03-21T12:44:44.984 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:44 smithi022 ceph-mon[67047]: pgmap v197: 97 pgs: 97 active+clean; 343 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.8 MiB/s wr, 555 op/s 2024-03-21T12:44:44.985 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:44 smithi022 ceph-mon[67047]: osd.4 marked itself down and dead 2024-03-21T12:44:45.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:44 smithi189 ceph-mon[53160]: pgmap v197: 97 pgs: 97 active+clean; 343 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.8 MiB/s wr, 555 op/s 2024-03-21T12:44:45.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:44 smithi189 ceph-mon[53160]: osd.4 marked itself down and dead 2024-03-21T12:44:45.627 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 21 12:44:45 smithi189 podman[57628]: 2024-03-21 12:44:45.349869019 +0000 UTC m=+0.951355661 container died 1bedc4eb02ed02c479eac0eca7422787f670bb5be7df6ea2a362d382f29b6344 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-4, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.tags=base centos centos-stream, distribution-scope=public, architecture=x86_64, vcs-type=git, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, release=754, CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, ceph=True, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container) 2024-03-21T12:44:45.887 INFO:teuthology.orchestra.run.smithi022.stdout: 5 250799 10.26 MB/sec execute 465 sec latency 932.638 ms 2024-03-21T12:44:45.954 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 21 12:44:45 smithi189 podman[57628]: 2024-03-21 12:44:45.726307692 +0000 UTC m=+1.327794380 container cleanup 1bedc4eb02ed02c479eac0eca7422787f670bb5be7df6ea2a362d382f29b6344 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-4, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, io.buildah.version=1.19.8, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, version=8, vendor=Red Hat, Inc., GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, release=754) 2024-03-21T12:44:45.955 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 21 12:44:45 smithi189 bash[57628]: ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-4 2024-03-21T12:44:46.205 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:45 smithi022 ceph-mon[67047]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-21T12:44:46.205 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:45 smithi022 ceph-mon[67047]: osdmap e120: 8 total, 7 up, 8 in 2024-03-21T12:44:46.205 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:45 smithi022 ceph-mon[67047]: osdmap e121: 8 total, 7 up, 8 in 2024-03-21T12:44:46.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:45 smithi189 ceph-mon[53160]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-21T12:44:46.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:45 smithi189 ceph-mon[53160]: osdmap e120: 8 total, 7 up, 8 in 2024-03-21T12:44:46.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:45 smithi189 ceph-mon[53160]: osdmap e121: 8 total, 7 up, 8 in 2024-03-21T12:44:46.286 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 21 12:44:45 smithi189 podman[57645]: 2024-03-21 12:44:45.968913517 +0000 UTC m=+0.615691644 container remove 1bedc4eb02ed02c479eac0eca7422787f670bb5be7df6ea2a362d382f29b6344 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-4, GIT_CLEAN=True, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, GIT_BRANCH=HEAD, ceph=True, com.redhat.component=centos-stream-container, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.tags=base centos centos-stream, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0) 2024-03-21T12:44:46.536 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 21 12:44:46 smithi189 podman[57734]: 2024-03-21 12:44:46.190210116 +0000 UTC m=+0.019698436 image pull 19e12293512ff7978c8adb5f72a829060dd47f07bfd6903b6b3178d2ebc29e4e quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93 2024-03-21T12:44:46.536 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 21 12:44:46 smithi189 podman[57734]: 2024-03-21 12:44:46.328447471 +0000 UTC m=+0.157935813 container create 34d70fafdcc1f0459f1deb00cc1d782cd320482126378c6d9a10514b82ac9936 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-4-deactivate, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.vendor=CentOS, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, org.label-schema.build-date=20240319, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-03-21T12:44:46.887 INFO:teuthology.orchestra.run.smithi022.stdout: 5 250973 10.25 MB/sec execute 466 sec latency 1020.831 ms 2024-03-21T12:44:46.889 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 21 12:44:46 smithi189 podman[57734]: 2024-03-21 12:44:46.575719127 +0000 UTC m=+0.405207451 container init 34d70fafdcc1f0459f1deb00cc1d782cd320482126378c6d9a10514b82ac9936 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-4-deactivate, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240319, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True) 2024-03-21T12:44:46.889 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 21 12:44:46 smithi189 podman[57734]: 2024-03-21 12:44:46.57881504 +0000 UTC m=+0.408303342 container start 34d70fafdcc1f0459f1deb00cc1d782cd320482126378c6d9a10514b82ac9936 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-4-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, org.label-schema.build-date=20240319) 2024-03-21T12:44:46.889 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 21 12:44:46 smithi189 podman[57734]: 2024-03-21 12:44:46.68097826 +0000 UTC m=+0.510466563 container attach 34d70fafdcc1f0459f1deb00cc1d782cd320482126378c6d9a10514b82ac9936 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-4-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240319, io.buildah.version=1.33.5, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2) 2024-03-21T12:44:47.201 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:46 smithi189 ceph-mon[53160]: pgmap v199: 97 pgs: 14 stale+active+clean, 83 active+clean; 343 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.7 MiB/s wr, 468 op/s 2024-03-21T12:44:47.201 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 21 12:44:46 smithi189 podman[57734]: 2024-03-21 12:44:46.890356255 +0000 UTC m=+0.719844559 container died 34d70fafdcc1f0459f1deb00cc1d782cd320482126378c6d9a10514b82ac9936 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-4-deactivate, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240319, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-03-21T12:44:47.211 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:46 smithi022 ceph-mon[67047]: pgmap v199: 97 pgs: 14 stale+active+clean, 83 active+clean; 343 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.7 MiB/s wr, 468 op/s 2024-03-21T12:44:47.786 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 21 12:44:47 smithi189 podman[57760]: 2024-03-21 12:44:47.524130986 +0000 UTC m=+0.624338756 container remove 34d70fafdcc1f0459f1deb00cc1d782cd320482126378c6d9a10514b82ac9936 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-4-deactivate, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.build-date=20240319, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux ) 2024-03-21T12:44:47.786 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 21 12:44:47 smithi189 systemd[1]: ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715@osd.4.service: Deactivated successfully. 2024-03-21T12:44:47.786 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 21 12:44:47 smithi189 systemd[1]: Stopped Ceph osd.4 for 3fc7f0f2-e77e-11ee-95cd-87774f69a715. 2024-03-21T12:44:47.786 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 21 12:44:47 smithi189 systemd[1]: ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715@osd.4.service: Consumed 4min 49.675s CPU time. 2024-03-21T12:44:47.887 INFO:teuthology.orchestra.run.smithi022.stdout: 5 251192 10.24 MB/sec execute 467 sec latency 109.152 ms 2024-03-21T12:44:48.183 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:48 smithi189 ceph-mon[53160]: Health check failed: Degraded data redundancy: 185/1566 objects degraded (11.814%), 23 pgs degraded (PG_DEGRADED) 2024-03-21T12:44:48.183 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 21 12:44:47 smithi189 systemd[1]: Starting Ceph osd.4 for 3fc7f0f2-e77e-11ee-95cd-87774f69a715... 2024-03-21T12:44:48.183 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 21 12:44:48 smithi189 podman[57863]: 2024-03-21 12:44:48.052721262 +0000 UTC m=+0.105598925 container create 50c0d2e1db101b7c3d3724cd48c11fb6a62592b3c6ff35f021b6783247964894 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-4-activate, org.label-schema.build-date=20240319, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-21T12:44:48.183 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 21 12:44:48 smithi189 podman[57863]: 2024-03-21 12:44:47.965356859 +0000 UTC m=+0.018234523 image pull 19e12293512ff7978c8adb5f72a829060dd47f07bfd6903b6b3178d2ebc29e4e quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93 2024-03-21T12:44:48.230 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:48 smithi022 ceph-mon[67047]: Health check failed: Degraded data redundancy: 185/1566 objects degraded (11.814%), 23 pgs degraded (PG_DEGRADED) 2024-03-21T12:44:48.536 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 21 12:44:48 smithi189 podman[57863]: 2024-03-21 12:44:48.322585438 +0000 UTC m=+0.375463121 container init 50c0d2e1db101b7c3d3724cd48c11fb6a62592b3c6ff35f021b6783247964894 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-4-activate, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240319, ceph=True, io.buildah.version=1.33.5, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, org.label-schema.license=GPLv2) 2024-03-21T12:44:48.536 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 21 12:44:48 smithi189 podman[57863]: 2024-03-21 12:44:48.325816981 +0000 UTC m=+0.378694653 container start 50c0d2e1db101b7c3d3724cd48c11fb6a62592b3c6ff35f021b6783247964894 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-4-activate, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240319, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, ceph=True) 2024-03-21T12:44:48.536 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 21 12:44:48 smithi189 podman[57863]: 2024-03-21 12:44:48.422007646 +0000 UTC m=+0.474885314 container attach 50c0d2e1db101b7c3d3724cd48c11fb6a62592b3c6ff35f021b6783247964894 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-4-activate, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240319, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe) 2024-03-21T12:44:48.887 INFO:teuthology.orchestra.run.smithi022.stdout: 5 251564 10.24 MB/sec execute 468 sec latency 52.921 ms 2024-03-21T12:44:49.208 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:49 smithi189 ceph-mon[53160]: pgmap v201: 97 pgs: 7 active+undersized, 23 active+undersized+degraded, 5 stale+active+clean, 62 active+clean; 336 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 4.6 MiB/s wr, 458 op/s; 185/1566 objects degraded (11.814%) 2024-03-21T12:44:49.208 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 21 12:44:48 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-4-activate[57878]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-03-21T12:44:49.209 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 21 12:44:48 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-4-activate[57878]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-03-21T12:44:49.209 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 21 12:44:48 smithi189 bash[57863]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-03-21T12:44:49.209 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 21 12:44:48 smithi189 bash[57863]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-03-21T12:44:49.209 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 21 12:44:48 smithi189 bash[57863]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-03-21T12:44:49.209 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 21 12:44:48 smithi189 bash[57863]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-03-21T12:44:49.209 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 21 12:44:48 smithi189 bash[57863]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-03-21T12:44:49.209 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 21 12:44:48 smithi189 bash[57863]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-03-21T12:44:49.209 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 21 12:44:48 smithi189 bash[57863]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-03-21T12:44:49.209 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 21 12:44:48 smithi189 bash[57863]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-03-21T12:44:49.209 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 21 12:44:48 smithi189 bash[57863]: --> ceph-volume raw activate successful for osd ID: 4 2024-03-21T12:44:49.209 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 21 12:44:48 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-4-activate[57878]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-03-21T12:44:49.209 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 21 12:44:48 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-4-activate[57878]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-03-21T12:44:49.209 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 21 12:44:48 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-4-activate[57878]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-03-21T12:44:49.209 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 21 12:44:48 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-4-activate[57878]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-03-21T12:44:49.209 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 21 12:44:48 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-4-activate[57878]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-03-21T12:44:49.209 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 21 12:44:48 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-4-activate[57878]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-03-21T12:44:49.210 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 21 12:44:48 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-4-activate[57878]: --> ceph-volume raw activate successful for osd ID: 4 2024-03-21T12:44:49.210 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 21 12:44:48 smithi189 podman[57863]: 2024-03-21 12:44:48.881144154 +0000 UTC m=+0.934021832 container died 50c0d2e1db101b7c3d3724cd48c11fb6a62592b3c6ff35f021b6783247964894 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-4-activate, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240319, org.label-schema.vendor=CentOS, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.schema-version=1.0, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, ceph=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-03-21T12:44:49.240 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:49 smithi022 ceph-mon[67047]: pgmap v201: 97 pgs: 7 active+undersized, 23 active+undersized+degraded, 5 stale+active+clean, 62 active+clean; 336 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 4.6 MiB/s wr, 458 op/s; 185/1566 objects degraded (11.814%) 2024-03-21T12:44:49.887 INFO:teuthology.orchestra.run.smithi022.stdout: 5 252023 10.24 MB/sec execute 469 sec latency 32.302 ms 2024-03-21T12:44:50.019 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 21 12:44:49 smithi189 podman[58001]: 2024-03-21 12:44:49.650264982 +0000 UTC m=+0.761495966 container remove 50c0d2e1db101b7c3d3724cd48c11fb6a62592b3c6ff35f021b6783247964894 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-4-activate, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.build-date=20240319) 2024-03-21T12:44:50.019 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 21 12:44:49 smithi189 podman[58060]: 2024-03-21 12:44:49.775587377 +0000 UTC m=+0.021466327 image pull 19e12293512ff7978c8adb5f72a829060dd47f07bfd6903b6b3178d2ebc29e4e quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93 2024-03-21T12:44:50.019 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 21 12:44:49 smithi189 podman[58060]: 2024-03-21 12:44:49.894469859 +0000 UTC m=+0.140348788 container create 382755cee998bbface39691dbccd0c2f5550e133b6e9d4e9768f2935ef512d1c (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-4, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240319, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-03-21T12:44:50.286 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 21 12:44:50 smithi189 podman[58060]: 2024-03-21 12:44:50.156676943 +0000 UTC m=+0.402555887 container init 382755cee998bbface39691dbccd0c2f5550e133b6e9d4e9768f2935ef512d1c (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-4, org.label-schema.build-date=20240319, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2) 2024-03-21T12:44:50.286 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 21 12:44:50 smithi189 podman[58060]: 2024-03-21 12:44:50.160203401 +0000 UTC m=+0.406082335 container start 382755cee998bbface39691dbccd0c2f5550e133b6e9d4e9768f2935ef512d1c (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-4, org.label-schema.build-date=20240319, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-03-21T12:44:50.286 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 21 12:44:50 smithi189 ceph-osd[58079]: -- 172.21.15.189:0/722937021 <== mon.1 v2:172.21.15.189:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5622938b3680 con 0x56229387d000 2024-03-21T12:44:50.286 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 21 12:44:50 smithi189 bash[58060]: 382755cee998bbface39691dbccd0c2f5550e133b6e9d4e9768f2935ef512d1c 2024-03-21T12:44:50.286 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 21 12:44:50 smithi189 systemd[1]: Started Ceph osd.4 for 3fc7f0f2-e77e-11ee-95cd-87774f69a715. 2024-03-21T12:44:50.887 INFO:teuthology.orchestra.run.smithi022.stdout: 5 252395 10.25 MB/sec execute 470 sec latency 72.367 ms 2024-03-21T12:44:51.036 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 21 12:44:50 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-4[58075]: 2024-03-21T12:44:50.697+0000 7f88dafa97c0 -1 Falling back to public interface 2024-03-21T12:44:51.527 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:51 smithi022 ceph-mon[67047]: pgmap v202: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 334 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.0 MiB/s wr, 479 op/s; 235/1515 objects degraded (15.512%) 2024-03-21T12:44:51.527 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:51 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:44:51.527 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:51 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:44:51.527 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:51 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:44:51.527 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:51 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:44:51.527 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:51 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:44:51.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:51 smithi189 ceph-mon[53160]: pgmap v202: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 334 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.0 MiB/s wr, 479 op/s; 235/1515 objects degraded (15.512%) 2024-03-21T12:44:51.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:51 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:44:51.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:51 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:44:51.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:51 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:44:51.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:51 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:44:51.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:51 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:44:51.887 INFO:teuthology.orchestra.run.smithi022.stdout: 5 252805 10.25 MB/sec execute 471 sec latency 35.275 ms 2024-03-21T12:44:52.887 INFO:teuthology.orchestra.run.smithi022.stdout: 5 253158 10.25 MB/sec execute 472 sec latency 41.869 ms 2024-03-21T12:44:53.551 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:53 smithi022 ceph-mon[67047]: pgmap v203: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 334 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4.4 KiB/s rd, 1.8 MiB/s wr, 193 op/s; 235/1515 objects degraded (15.512%) 2024-03-21T12:44:53.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:53 smithi189 ceph-mon[53160]: pgmap v203: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 334 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4.4 KiB/s rd, 1.8 MiB/s wr, 193 op/s; 235/1515 objects degraded (15.512%) 2024-03-21T12:44:53.887 INFO:teuthology.orchestra.run.smithi022.stdout: 5 253432 10.24 MB/sec execute 473 sec latency 49.851 ms 2024-03-21T12:44:54.475 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:54 smithi189 ceph-mon[53160]: Health check update: Degraded data redundancy: 235/1515 objects degraded (15.512%), 28 pgs degraded (PG_DEGRADED) 2024-03-21T12:44:54.568 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:54 smithi022 ceph-mon[67047]: Health check update: Degraded data redundancy: 235/1515 objects degraded (15.512%), 28 pgs degraded (PG_DEGRADED) 2024-03-21T12:44:54.887 INFO:teuthology.orchestra.run.smithi022.stdout: 5 253765 10.25 MB/sec execute 474 sec latency 47.322 ms 2024-03-21T12:44:55.579 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:55 smithi022 ceph-mon[67047]: pgmap v204: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 337 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.8 MiB/s wr, 526 op/s; 259/1662 objects degraded (15.584%) 2024-03-21T12:44:55.580 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:55 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:44:55.580 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:55 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:44:55.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:55 smithi189 ceph-mon[53160]: pgmap v204: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 337 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.8 MiB/s wr, 526 op/s; 259/1662 objects degraded (15.584%) 2024-03-21T12:44:55.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:55 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:44:55.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:55 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:44:55.887 INFO:teuthology.orchestra.run.smithi022.stdout: 5 254052 10.23 MB/sec execute 475 sec latency 42.750 ms 2024-03-21T12:44:56.863 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:56 smithi022 ceph-mon[67047]: pgmap v205: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 337 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.2 MiB/s wr, 455 op/s; 259/1662 objects degraded (15.584%) 2024-03-21T12:44:56.864 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:56 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:44:56.864 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:56 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:44:56.887 INFO:teuthology.orchestra.run.smithi022.stdout: 5 254289 10.23 MB/sec execute 476 sec latency 46.014 ms 2024-03-21T12:44:57.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:56 smithi189 ceph-mon[53160]: pgmap v205: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 337 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.2 MiB/s wr, 455 op/s; 259/1662 objects degraded (15.584%) 2024-03-21T12:44:57.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:56 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:44:57.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:56 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:44:57.722 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 21 12:44:57 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-4[58075]: 2024-03-21T12:44:57.418+0000 7f88dafa97c0 -1 osd.4 0 read_superblock omap replica is missing. 2024-03-21T12:44:57.887 INFO:teuthology.orchestra.run.smithi022.stdout: 5 254500 10.22 MB/sec execute 477 sec latency 80.486 ms 2024-03-21T12:44:58.887 INFO:teuthology.orchestra.run.smithi022.stdout: 5 254737 10.21 MB/sec execute 478 sec latency 66.641 ms 2024-03-21T12:44:58.932 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:58 smithi022 ceph-mon[67047]: pgmap v206: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 340 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 4.8 MiB/s wr, 489 op/s; 264/1635 objects degraded (16.147%) 2024-03-21T12:44:59.030 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:58 smithi189 ceph-mon[53160]: pgmap v206: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 340 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 4.8 MiB/s wr, 489 op/s; 264/1635 objects degraded (16.147%) 2024-03-21T12:44:59.285 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 21 12:44:59 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-4[58075]: 2024-03-21T12:44:59.029+0000 7f88dafa97c0 -1 osd.4 119 log_to_monitors true 2024-03-21T12:44:59.887 INFO:teuthology.orchestra.run.smithi022.stdout: 5 255088 10.20 MB/sec execute 479 sec latency 70.852 ms 2024-03-21T12:44:59.965 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:59 smithi022 ceph-mon[67047]: Health check update: Degraded data redundancy: 264/1635 objects degraded (16.147%), 28 pgs degraded (PG_DEGRADED) 2024-03-21T12:44:59.965 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:59 smithi022 ceph-mon[67047]: from='osd.4 [v2:172.21.15.189:6800/91079179,v1:172.21.15.189:6801/91079179]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-03-21T12:44:59.965 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:44:59 smithi022 ceph-mon[67047]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-03-21T12:45:00.036 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 21 12:44:59 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-4[58075]: 2024-03-21T12:44:59.793+0000 7f88cc401700 -1 osd.4 119 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-21T12:45:00.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:59 smithi189 ceph-mon[53160]: Health check update: Degraded data redundancy: 264/1635 objects degraded (16.147%), 28 pgs degraded (PG_DEGRADED) 2024-03-21T12:45:00.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:59 smithi189 ceph-mon[53160]: from='osd.4 [v2:172.21.15.189:6800/91079179,v1:172.21.15.189:6801/91079179]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-03-21T12:45:00.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:44:59 smithi189 ceph-mon[53160]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-03-21T12:45:00.887 INFO:teuthology.orchestra.run.smithi022.stdout: 5 255407 10.20 MB/sec execute 480 sec latency 71.210 ms 2024-03-21T12:45:01.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:00 smithi189 ceph-mon[53160]: pgmap v207: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 341 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.2 MiB/s wr, 413 op/s; 268/1632 objects degraded (16.422%) 2024-03-21T12:45:01.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:00 smithi189 ceph-mon[53160]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-03-21T12:45:01.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:00 smithi189 ceph-mon[53160]: osdmap e122: 8 total, 7 up, 8 in 2024-03-21T12:45:01.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:00 smithi189 ceph-mon[53160]: from='osd.4 [v2:172.21.15.189:6800/91079179,v1:172.21.15.189:6801/91079179]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-03-21T12:45:01.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:00 smithi189 ceph-mon[53160]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-03-21T12:45:01.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:00 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:45:01.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:00 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:45:01.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:00 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:45:01.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:00 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:45:01.037 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:00 smithi189 ceph-mon[53160]: pgmap v209: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 341 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.7 MiB/s wr, 579 op/s; 268/1632 objects degraded (16.422%) 2024-03-21T12:45:01.037 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:00 smithi189 ceph-mon[53160]: pgmap v210: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 341 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 3.3 MiB/s wr, 283 op/s; 268/1632 objects degraded (16.422%) 2024-03-21T12:45:01.037 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:00 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:45:01.037 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:00 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:45:01.037 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:00 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:45:01.037 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:00 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:45:01.037 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:00 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:45:01.037 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:00 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:45:01.037 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:00 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-03-21T12:45:01.037 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:00 smithi189 ceph-mon[53160]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-03-21T12:45:01.037 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:00 smithi189 ceph-mon[53160]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-03-21T12:45:01.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:00 smithi022 ceph-mon[67047]: pgmap v207: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 341 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.2 MiB/s wr, 413 op/s; 268/1632 objects degraded (16.422%) 2024-03-21T12:45:01.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:00 smithi022 ceph-mon[67047]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-03-21T12:45:01.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:00 smithi022 ceph-mon[67047]: osdmap e122: 8 total, 7 up, 8 in 2024-03-21T12:45:01.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:00 smithi022 ceph-mon[67047]: from='osd.4 [v2:172.21.15.189:6800/91079179,v1:172.21.15.189:6801/91079179]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-03-21T12:45:01.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:00 smithi022 ceph-mon[67047]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-03-21T12:45:01.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:00 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:45:01.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:00 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:45:01.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:00 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:45:01.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:00 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:45:01.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:00 smithi022 ceph-mon[67047]: pgmap v209: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 341 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.7 MiB/s wr, 579 op/s; 268/1632 objects degraded (16.422%) 2024-03-21T12:45:01.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:00 smithi022 ceph-mon[67047]: pgmap v210: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 341 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 3.3 MiB/s wr, 283 op/s; 268/1632 objects degraded (16.422%) 2024-03-21T12:45:01.096 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:00 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:45:01.096 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:00 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:45:01.096 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:00 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:45:01.096 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:00 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:45:01.096 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:00 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:45:01.096 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:00 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:45:01.096 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:00 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-03-21T12:45:01.096 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:00 smithi022 ceph-mon[67047]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-03-21T12:45:01.096 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:00 smithi022 ceph-mon[67047]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-03-21T12:45:01.887 INFO:teuthology.orchestra.run.smithi022.stdout: 5 255408 10.18 MB/sec execute 481 sec latency 1071.241 ms 2024-03-21T12:45:02.017 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:01 smithi022 ceph-mon[67047]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-21T12:45:02.018 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:01 smithi022 ceph-mon[67047]: osd.4 [v2:172.21.15.189:6800/91079179,v1:172.21.15.189:6801/91079179] boot 2024-03-21T12:45:02.018 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:01 smithi022 ceph-mon[67047]: osdmap e123: 8 total, 8 up, 8 in 2024-03-21T12:45:02.018 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:01 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-21T12:45:02.018 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:01 smithi022 ceph-mon[67047]: osdmap e124: 8 total, 8 up, 8 in 2024-03-21T12:45:02.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:01 smithi189 ceph-mon[53160]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-21T12:45:02.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:01 smithi189 ceph-mon[53160]: osd.4 [v2:172.21.15.189:6800/91079179,v1:172.21.15.189:6801/91079179] boot 2024-03-21T12:45:02.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:01 smithi189 ceph-mon[53160]: osdmap e123: 8 total, 8 up, 8 in 2024-03-21T12:45:02.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:01 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-21T12:45:02.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:01 smithi189 ceph-mon[53160]: osdmap e124: 8 total, 8 up, 8 in 2024-03-21T12:45:02.816 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:02 smithi022 ceph-mon[67047]: pgmap v213: 97 pgs: 14 peering, 13 active+undersized, 14 active+undersized+degraded, 56 active+clean; 338 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 15 KiB/s rd, 3.3 MiB/s wr, 389 op/s; 136/1590 objects degraded (8.553%) 2024-03-21T12:45:02.816 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:02 smithi022 ceph-mon[67047]: osdmap e125: 8 total, 8 up, 8 in 2024-03-21T12:45:02.887 INFO:teuthology.orchestra.run.smithi022.stdout: 5 255587 10.17 MB/sec execute 482 sec latency 1956.282 ms 2024-03-21T12:45:03.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:02 smithi189 ceph-mon[53160]: pgmap v213: 97 pgs: 14 peering, 13 active+undersized, 14 active+undersized+degraded, 56 active+clean; 338 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 15 KiB/s rd, 3.3 MiB/s wr, 389 op/s; 136/1590 objects degraded (8.553%) 2024-03-21T12:45:03.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:02 smithi189 ceph-mon[53160]: osdmap e125: 8 total, 8 up, 8 in 2024-03-21T12:45:03.849 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:03 smithi022 ceph-mon[67047]: Health check update: Degraded data redundancy: 136/1590 objects degraded (8.553%), 14 pgs degraded (PG_DEGRADED) 2024-03-21T12:45:03.850 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:03 smithi022 ceph-mon[67047]: osdmap e126: 8 total, 8 up, 8 in 2024-03-21T12:45:03.887 INFO:teuthology.orchestra.run.smithi022.stdout: 5 255891 10.17 MB/sec execute 483 sec latency 120.185 ms 2024-03-21T12:45:04.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:03 smithi189 ceph-mon[53160]: Health check update: Degraded data redundancy: 136/1590 objects degraded (8.553%), 14 pgs degraded (PG_DEGRADED) 2024-03-21T12:45:04.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:03 smithi189 ceph-mon[53160]: osdmap e126: 8 total, 8 up, 8 in 2024-03-21T12:45:04.854 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:04 smithi022 ceph-mon[67047]: pgmap v216: 97 pgs: 1 active+recovery_wait+degraded, 1 remapped+peering, 14 peering, 9 active+undersized, 5 active+undersized+degraded, 67 active+clean; 338 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 3.5 MiB/s wr, 404 op/s; 69/1539 objects degraded (4.483%); 118 KiB/s, 58 keys/s, 12 objects/s recovering 2024-03-21T12:45:04.854 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:04 smithi022 ceph-mon[67047]: osdmap e127: 8 total, 8 up, 8 in 2024-03-21T12:45:04.887 INFO:teuthology.orchestra.run.smithi022.stdout: 5 256276 10.17 MB/sec execute 484 sec latency 1033.261 ms 2024-03-21T12:45:05.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:04 smithi189 ceph-mon[53160]: pgmap v216: 97 pgs: 1 active+recovery_wait+degraded, 1 remapped+peering, 14 peering, 9 active+undersized, 5 active+undersized+degraded, 67 active+clean; 338 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 3.5 MiB/s wr, 404 op/s; 69/1539 objects degraded (4.483%); 118 KiB/s, 58 keys/s, 12 objects/s recovering 2024-03-21T12:45:05.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:04 smithi189 ceph-mon[53160]: osdmap e127: 8 total, 8 up, 8 in 2024-03-21T12:45:05.476 INFO:teuthology.orchestra.run.smithi022.stdout:true 2024-03-21T12:45:05.887 INFO:teuthology.orchestra.run.smithi022.stdout: 5 256616 10.16 MB/sec execute 485 sec latency 39.303 ms 2024-03-21T12:45:06.549 INFO:teuthology.orchestra.run.smithi022.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-21T12:45:06.549 INFO:teuthology.orchestra.run.smithi022.stdout:alertmanager.smithi022 smithi022 *:9093,9094 running (7m) 49s ago 16m 24.1M - 0.23.0 ba2b418f427c 14eb2a3ca318 2024-03-21T12:45:06.549 INFO:teuthology.orchestra.run.smithi022.stdout:crash.smithi022 smithi022 running (3m) 49s ago 16m 6660k - 19.0.0-2267-g960fbbe4 19e12293512f 356b2ddb609a 2024-03-21T12:45:06.549 INFO:teuthology.orchestra.run.smithi022.stdout:crash.smithi189 smithi189 running (3m) 12s ago 15m 6660k - 19.0.0-2267-g960fbbe4 19e12293512f 90ee837f0500 2024-03-21T12:45:06.549 INFO:teuthology.orchestra.run.smithi022.stdout:grafana.smithi022 smithi022 *:3000 error 49s ago 16m - - 2024-03-21T12:45:06.549 INFO:teuthology.orchestra.run.smithi022.stdout:mds.foofs.smithi022.bcnxho smithi022 running (10m) 49s ago 10m 38.3M - 17.2.0 e1d6a67b021e db26b67b8ee7 2024-03-21T12:45:06.549 INFO:teuthology.orchestra.run.smithi022.stdout:mds.foofs.smithi189.lluvko smithi189 running (10m) 12s ago 10m 17.6M - 17.2.0 e1d6a67b021e cf153aff1387 2024-03-21T12:45:06.549 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.smithi022.fektzs smithi022 *:8443,9283,8765 running (6m) 49s ago 17m 506M - 19.0.0-2267-g960fbbe4 19e12293512f d57cebd306b3 2024-03-21T12:45:06.549 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.smithi189.cnxzvu smithi189 *:8443,9283,8765 running (5m) 12s ago 15m 451M - 19.0.0-2267-g960fbbe4 19e12293512f 41663955c56f 2024-03-21T12:45:06.550 INFO:teuthology.orchestra.run.smithi022.stdout:mon.smithi022 smithi022 running (5m) 49s ago 17m 50.6M 2048M 19.0.0-2267-g960fbbe4 19e12293512f c620c9071b2c 2024-03-21T12:45:06.550 INFO:teuthology.orchestra.run.smithi022.stdout:mon.smithi189 smithi189 running (4m) 12s ago 15m 38.8M 2048M 19.0.0-2267-g960fbbe4 19e12293512f b353af3d8f3b 2024-03-21T12:45:06.550 INFO:teuthology.orchestra.run.smithi022.stdout:nfs.foo.0.0.smithi022.ynhbbq smithi022 *:2049 running (10m) 49s ago 10m 277M - 4.0 e1d6a67b021e 33e48e64c60b 2024-03-21T12:45:06.550 INFO:teuthology.orchestra.run.smithi022.stdout:nfs.foo.1.0.smithi189.yzsqml smithi189 *:2049 running (10m) 12s ago 10m 85.9M - 4.0 e1d6a67b021e 48fab2cbdde2 2024-03-21T12:45:06.550 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.smithi022 smithi022 *:9100 running (7m) 49s ago 16m 18.2M - 1.3.1 1dbe0e931976 33287f803f4f 2024-03-21T12:45:06.550 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.smithi189 smithi189 *:9100 running (6m) 12s ago 15m 17.3M - 1.3.1 1dbe0e931976 b51d872e438a 2024-03-21T12:45:06.550 INFO:teuthology.orchestra.run.smithi022.stdout:osd.0 smithi022 running (3m) 49s ago 15m 336M 1937M 19.0.0-2267-g960fbbe4 19e12293512f 23a85645eeea 2024-03-21T12:45:06.550 INFO:teuthology.orchestra.run.smithi022.stdout:osd.1 smithi022 running (2m) 49s ago 14m 286M 1937M 19.0.0-2267-g960fbbe4 19e12293512f 0f90e70bbf0a 2024-03-21T12:45:06.550 INFO:teuthology.orchestra.run.smithi022.stdout:osd.2 smithi022 running (107s) 49s ago 14m 214M 1937M 19.0.0-2267-g960fbbe4 19e12293512f 78a229f593f8 2024-03-21T12:45:06.550 INFO:teuthology.orchestra.run.smithi022.stdout:osd.3 smithi022 running (57s) 49s ago 13m 12.5M 1937M 19.0.0-2267-g960fbbe4 19e12293512f 61a63b67d9af 2024-03-21T12:45:06.550 INFO:teuthology.orchestra.run.smithi022.stdout:osd.4 smithi189 running (16s) 12s ago 13m 12.0M 2704M 19.0.0-2267-g960fbbe4 19e12293512f 382755cee998 2024-03-21T12:45:06.550 INFO:teuthology.orchestra.run.smithi022.stdout:osd.5 smithi189 running (12m) 12s ago 12m 591M 2704M 17.2.0 e1d6a67b021e 25fbab9786a5 2024-03-21T12:45:06.550 INFO:teuthology.orchestra.run.smithi022.stdout:osd.6 smithi189 running (12m) 12s ago 12m 518M 2704M 17.2.0 e1d6a67b021e 0c192fcfd65b 2024-03-21T12:45:06.550 INFO:teuthology.orchestra.run.smithi022.stdout:osd.7 smithi189 running (12m) 12s ago 12m 571M 2704M 17.2.0 e1d6a67b021e 3d0723e1f863 2024-03-21T12:45:06.551 INFO:teuthology.orchestra.run.smithi022.stdout:prometheus.smithi022 smithi022 *:9095 running (5m) 49s ago 15m 73.9M - 2.33.4 514e6a882f6e 8171bbe41d2c 2024-03-21T12:45:06.885 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:06 smithi022 ceph-mon[67047]: from='client.34360 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:45:06.885 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:06 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:45:06.885 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:06 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:45:06.885 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:06 smithi022 ceph-mon[67047]: osdmap e128: 8 total, 8 up, 8 in 2024-03-21T12:45:06.885 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:06 smithi022 ceph-mon[67047]: pgmap v219: 97 pgs: 1 active+recovery_wait+degraded, 1 remapped+peering, 14 peering, 81 active+clean; 338 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 1.9 MiB/s wr, 207 op/s; 21/1539 objects degraded (1.365%); 254 KiB/s, 58 keys/s, 29 objects/s recovering 2024-03-21T12:45:06.885 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:06 smithi022 ceph-mon[67047]: from='client.34364 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:45:06.887 INFO:teuthology.orchestra.run.smithi022.stdout: 5 256894 10.16 MB/sec execute 486 sec latency 45.347 ms 2024-03-21T12:45:07.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:06 smithi189 ceph-mon[53160]: from='client.34360 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:45:07.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:06 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:45:07.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:06 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:45:07.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:06 smithi189 ceph-mon[53160]: osdmap e128: 8 total, 8 up, 8 in 2024-03-21T12:45:07.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:06 smithi189 ceph-mon[53160]: pgmap v219: 97 pgs: 1 active+recovery_wait+degraded, 1 remapped+peering, 14 peering, 81 active+clean; 338 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 1.9 MiB/s wr, 207 op/s; 21/1539 objects degraded (1.365%); 254 KiB/s, 58 keys/s, 29 objects/s recovering 2024-03-21T12:45:07.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:06 smithi189 ceph-mon[53160]: from='client.34364 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:45:07.229 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-03-21T12:45:07.229 INFO:teuthology.orchestra.run.smithi022.stdout: "mon": { 2024-03-21T12:45:07.229 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 2 2024-03-21T12:45:07.229 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:45:07.229 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr": { 2024-03-21T12:45:07.229 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 2 2024-03-21T12:45:07.229 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:45:07.229 INFO:teuthology.orchestra.run.smithi022.stdout: "osd": { 2024-03-21T12:45:07.229 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-03-21T12:45:07.230 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 5 2024-03-21T12:45:07.230 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:45:07.230 INFO:teuthology.orchestra.run.smithi022.stdout: "mds": { 2024-03-21T12:45:07.230 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-21T12:45:07.230 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:45:07.230 INFO:teuthology.orchestra.run.smithi022.stdout: "overall": { 2024-03-21T12:45:07.230 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-03-21T12:45:07.230 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 9 2024-03-21T12:45:07.230 INFO:teuthology.orchestra.run.smithi022.stdout: } 2024-03-21T12:45:07.230 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-03-21T12:45:07.723 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-03-21T12:45:07.723 INFO:teuthology.orchestra.run.smithi022.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93", 2024-03-21T12:45:07.723 INFO:teuthology.orchestra.run.smithi022.stdout: "in_progress": true, 2024-03-21T12:45:07.723 INFO:teuthology.orchestra.run.smithi022.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-21T12:45:07.723 INFO:teuthology.orchestra.run.smithi022.stdout: "services_complete": [ 2024-03-21T12:45:07.723 INFO:teuthology.orchestra.run.smithi022.stdout: "mon", 2024-03-21T12:45:07.723 INFO:teuthology.orchestra.run.smithi022.stdout: "crash", 2024-03-21T12:45:07.723 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr" 2024-03-21T12:45:07.723 INFO:teuthology.orchestra.run.smithi022.stdout: ], 2024-03-21T12:45:07.724 INFO:teuthology.orchestra.run.smithi022.stdout: "progress": "11/23 daemons upgraded", 2024-03-21T12:45:07.724 INFO:teuthology.orchestra.run.smithi022.stdout: "message": "Currently upgrading osd daemons", 2024-03-21T12:45:07.724 INFO:teuthology.orchestra.run.smithi022.stdout: "is_paused": false 2024-03-21T12:45:07.724 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-03-21T12:45:07.887 INFO:teuthology.orchestra.run.smithi022.stdout: 5 257316 10.16 MB/sec execute 487 sec latency 51.862 ms 2024-03-21T12:45:07.920 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:07 smithi022 ceph-mon[67047]: from='client.34368 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:45:07.920 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:07 smithi022 ceph-mon[67047]: osdmap e129: 8 total, 8 up, 8 in 2024-03-21T12:45:07.921 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:07 smithi022 ceph-mon[67047]: from='client.? 172.21.15.22:0/144736597' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:45:08.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:07 smithi189 ceph-mon[53160]: from='client.34368 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:45:08.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:07 smithi189 ceph-mon[53160]: osdmap e129: 8 total, 8 up, 8 in 2024-03-21T12:45:08.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:07 smithi189 ceph-mon[53160]: from='client.? 172.21.15.22:0/144736597' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:45:08.292 INFO:teuthology.orchestra.run.smithi022.stdout:HEALTH_WARN 1 failed cephadm daemon(s); Degraded data redundancy: 21/1539 objects degraded (1.365%), 1 pg degraded 2024-03-21T12:45:08.292 INFO:teuthology.orchestra.run.smithi022.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-21T12:45:08.292 INFO:teuthology.orchestra.run.smithi022.stdout: daemon grafana.smithi022 on smithi022 is in error state 2024-03-21T12:45:08.292 INFO:teuthology.orchestra.run.smithi022.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 21/1539 objects degraded (1.365%), 1 pg degraded 2024-03-21T12:45:08.293 INFO:teuthology.orchestra.run.smithi022.stdout: pg 3.1b is active+recovery_wait+degraded, acting [0,4,7] 2024-03-21T12:45:08.887 INFO:teuthology.orchestra.run.smithi022.stdout: 5 257715 10.17 MB/sec execute 488 sec latency 35.389 ms 2024-03-21T12:45:08.932 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:08 smithi022 ceph-mon[67047]: from='client.34376 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:45:08.932 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:08 smithi022 ceph-mon[67047]: pgmap v221: 97 pgs: 2 peering, 95 active+clean; 341 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 5.6 MiB/s wr, 553 op/s; 1.3 MiB/s, 0 keys/s, 57 objects/s recovering 2024-03-21T12:45:08.933 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:08 smithi022 ceph-mon[67047]: from='client.? 172.21.15.22:0/1017445635' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-21T12:45:08.933 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:08 smithi022 ceph-mon[67047]: Health check update: Degraded data redundancy: 21/1539 objects degraded (1.365%), 1 pg degraded (PG_DEGRADED) 2024-03-21T12:45:09.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:08 smithi189 ceph-mon[53160]: from='client.34376 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:45:09.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:08 smithi189 ceph-mon[53160]: pgmap v221: 97 pgs: 2 peering, 95 active+clean; 341 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 5.6 MiB/s wr, 553 op/s; 1.3 MiB/s, 0 keys/s, 57 objects/s recovering 2024-03-21T12:45:09.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:08 smithi189 ceph-mon[53160]: from='client.? 172.21.15.22:0/1017445635' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-21T12:45:09.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:08 smithi189 ceph-mon[53160]: Health check update: Degraded data redundancy: 21/1539 objects degraded (1.365%), 1 pg degraded (PG_DEGRADED) 2024-03-21T12:45:09.887 INFO:teuthology.orchestra.run.smithi022.stdout: 5 257997 10.16 MB/sec execute 489 sec latency 57.853 ms 2024-03-21T12:45:09.940 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:09 smithi022 ceph-mon[67047]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 21/1539 objects degraded (1.365%), 1 pg degraded) 2024-03-21T12:45:10.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:09 smithi189 ceph-mon[53160]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 21/1539 objects degraded (1.365%), 1 pg degraded) 2024-03-21T12:45:10.887 INFO:teuthology.orchestra.run.smithi022.stdout: 5 258355 10.16 MB/sec execute 490 sec latency 40.375 ms 2024-03-21T12:45:10.993 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:10 smithi022 ceph-mon[67047]: pgmap v222: 97 pgs: 2 peering, 95 active+clean; 346 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.3 KiB/s rd, 5.1 MiB/s wr, 562 op/s; 898 KiB/s, 0 keys/s, 39 objects/s recovering 2024-03-21T12:45:11.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:10 smithi189 ceph-mon[53160]: pgmap v222: 97 pgs: 2 peering, 95 active+clean; 346 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.3 KiB/s rd, 5.1 MiB/s wr, 562 op/s; 898 KiB/s, 0 keys/s, 39 objects/s recovering 2024-03-21T12:45:11.887 INFO:teuthology.orchestra.run.smithi022.stdout: 5 258636 10.15 MB/sec execute 491 sec latency 85.123 ms 2024-03-21T12:45:12.888 INFO:teuthology.orchestra.run.smithi022.stdout: 5 258924 10.14 MB/sec execute 492 sec latency 53.906 ms 2024-03-21T12:45:13.077 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:12 smithi022 ceph-mon[67047]: pgmap v223: 97 pgs: 97 active+clean; 348 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.7 MiB/s wr, 588 op/s; 760 KiB/s, 0 keys/s, 33 objects/s recovering 2024-03-21T12:45:13.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:12 smithi189 ceph-mon[53160]: pgmap v223: 97 pgs: 97 active+clean; 348 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.7 MiB/s wr, 588 op/s; 760 KiB/s, 0 keys/s, 33 objects/s recovering 2024-03-21T12:45:13.887 INFO:teuthology.orchestra.run.smithi022.stdout: 5 259170 10.14 MB/sec execute 493 sec latency 78.883 ms 2024-03-21T12:45:14.888 INFO:teuthology.orchestra.run.smithi022.stdout: 5 259448 10.14 MB/sec execute 494 sec latency 81.615 ms 2024-03-21T12:45:15.156 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:14 smithi022 ceph-mon[67047]: pgmap v224: 97 pgs: 97 active+clean; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.7 MiB/s wr, 557 op/s; 600 KiB/s, 0 keys/s, 20 objects/s recovering 2024-03-21T12:45:15.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:14 smithi189 ceph-mon[53160]: pgmap v224: 97 pgs: 97 active+clean; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.7 MiB/s wr, 557 op/s; 600 KiB/s, 0 keys/s, 20 objects/s recovering 2024-03-21T12:45:15.888 INFO:teuthology.orchestra.run.smithi022.stdout: 5 259767 10.13 MB/sec execute 495 sec latency 95.963 ms 2024-03-21T12:45:15.942 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:15 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-03-21T12:45:15.942 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:15 smithi022 ceph-mon[67047]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-03-21T12:45:15.942 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:15 smithi022 ceph-mon[67047]: Upgrade: osd.5 is safe to restart 2024-03-21T12:45:15.985 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:15 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-03-21T12:45:15.985 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:15 smithi189 ceph-mon[53160]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-03-21T12:45:15.985 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:15 smithi189 ceph-mon[53160]: Upgrade: osd.5 is safe to restart 2024-03-21T12:45:16.887 INFO:teuthology.orchestra.run.smithi022.stdout: 5 260066 10.12 MB/sec execute 496 sec latency 39.579 ms 2024-03-21T12:45:16.958 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:16 smithi022 ceph-mon[67047]: pgmap v225: 97 pgs: 97 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.9 MiB/s wr, 474 op/s; 484 KiB/s, 0 keys/s, 16 objects/s recovering 2024-03-21T12:45:17.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:16 smithi189 ceph-mon[53160]: pgmap v225: 97 pgs: 97 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.9 MiB/s wr, 474 op/s; 484 KiB/s, 0 keys/s, 16 objects/s recovering 2024-03-21T12:45:17.888 INFO:teuthology.orchestra.run.smithi022.stdout: 5 260243 10.12 MB/sec execute 497 sec latency 76.532 ms 2024-03-21T12:45:18.280 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:18 smithi022 ceph-mon[67047]: Upgrade: Updating osd.5 2024-03-21T12:45:18.280 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:18 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:45:18.280 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:18 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-03-21T12:45:18.280 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:18 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:45:18.280 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:18 smithi022 ceph-mon[67047]: Deploying daemon osd.5 on smithi189 2024-03-21T12:45:18.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:18 smithi189 ceph-mon[53160]: Upgrade: Updating osd.5 2024-03-21T12:45:18.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:18 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:45:18.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:18 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-03-21T12:45:18.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:18 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:45:18.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:18 smithi189 ceph-mon[53160]: Deploying daemon osd.5 on smithi189 2024-03-21T12:45:18.888 INFO:teuthology.orchestra.run.smithi022.stdout: 5 260611 10.12 MB/sec execute 498 sec latency 56.672 ms 2024-03-21T12:45:19.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:19 smithi022 ceph-mon[67047]: pgmap v226: 97 pgs: 97 active+clean; 340 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 5.2 MiB/s wr, 528 op/s; 67 KiB/s, 3 objects/s recovering 2024-03-21T12:45:19.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:19 smithi189 ceph-mon[53160]: pgmap v226: 97 pgs: 97 active+clean; 340 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 5.2 MiB/s wr, 528 op/s; 67 KiB/s, 3 objects/s recovering 2024-03-21T12:45:19.888 INFO:teuthology.orchestra.run.smithi022.stdout: 5 260944 10.11 MB/sec execute 499 sec latency 78.339 ms 2024-03-21T12:45:20.888 INFO:teuthology.orchestra.run.smithi022.stdout: 5 261261 10.11 MB/sec execute 500 sec latency 48.374 ms 2024-03-21T12:45:21.393 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:21 smithi022 ceph-mon[67047]: pgmap v227: 97 pgs: 97 active+clean; 336 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.9 MiB/s wr, 386 op/s 2024-03-21T12:45:21.394 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:21 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:45:21.394 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:21 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:45:21.394 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:21 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:45:21.459 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:21 smithi189 ceph-mon[53160]: pgmap v227: 97 pgs: 97 active+clean; 336 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.9 MiB/s wr, 386 op/s 2024-03-21T12:45:21.459 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:21 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:45:21.459 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:21 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:45:21.459 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:21 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:45:21.786 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:45:21 smithi189 systemd[1]: Stopping Ceph osd.5 for 3fc7f0f2-e77e-11ee-95cd-87774f69a715... 2024-03-21T12:45:21.888 INFO:teuthology.orchestra.run.smithi022.stdout: 5 261485 10.11 MB/sec execute 501 sec latency 74.810 ms 2024-03-21T12:45:22.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:22 smithi189 ceph-mon[53160]: osd.5 marked itself down and dead 2024-03-21T12:45:22.286 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:45:22 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-5[34298]: 2024-03-21T12:45:22.019+0000 7fbd88881700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-21T12:45:22.286 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:45:22 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-5[34298]: 2024-03-21T12:45:22.019+0000 7fbd88881700 -1 osd.5 129 *** Got signal Terminated *** 2024-03-21T12:45:22.286 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:45:22 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-5[34298]: 2024-03-21T12:45:22.019+0000 7fbd88881700 -1 osd.5 129 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-21T12:45:22.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:22 smithi022 ceph-mon[67047]: osd.5 marked itself down and dead 2024-03-21T12:45:22.888 INFO:teuthology.orchestra.run.smithi022.stdout: 5 261550 10.09 MB/sec execute 502 sec latency 743.209 ms 2024-03-21T12:45:23.112 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:45:22 smithi189 podman[59892]: 2024-03-21 12:45:22.815251654 +0000 UTC m=+0.928422755 container died 25fbab9786a53619e2096b482b935b8d657666bb5db28409e99b188bd19ac47f (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-5, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, vcs-type=git, GIT_BRANCH=HEAD, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-03-21T12:45:23.413 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:23 smithi022 ceph-mon[67047]: pgmap v228: 97 pgs: 97 active+clean; 342 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 4.5 MiB/s wr, 443 op/s 2024-03-21T12:45:23.413 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:23 smithi022 ceph-mon[67047]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-21T12:45:23.413 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:23 smithi022 ceph-mon[67047]: osdmap e130: 8 total, 7 up, 8 in 2024-03-21T12:45:23.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:23 smithi189 ceph-mon[53160]: pgmap v228: 97 pgs: 97 active+clean; 342 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 4.5 MiB/s wr, 443 op/s 2024-03-21T12:45:23.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:23 smithi189 ceph-mon[53160]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-21T12:45:23.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:23 smithi189 ceph-mon[53160]: osdmap e130: 8 total, 7 up, 8 in 2024-03-21T12:45:23.536 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:45:23 smithi189 podman[59892]: 2024-03-21 12:45:23.311902543 +0000 UTC m=+1.425073644 container cleanup 25fbab9786a53619e2096b482b935b8d657666bb5db28409e99b188bd19ac47f (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-5, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, maintainer=Guillaume Abrioux , RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public, ceph=True, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, version=8) 2024-03-21T12:45:23.536 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:45:23 smithi189 bash[59892]: ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-5 2024-03-21T12:45:23.888 INFO:teuthology.orchestra.run.smithi022.stdout: 5 261790 10.08 MB/sec execute 503 sec latency 1055.230 ms 2024-03-21T12:45:23.898 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:45:23 smithi189 podman[59908]: 2024-03-21 12:45:23.546610271 +0000 UTC m=+0.726997558 container remove 25fbab9786a53619e2096b482b935b8d657666bb5db28409e99b188bd19ac47f (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-5, CEPH_POINT_RELEASE=-17.2.0, release=754, ceph=True, GIT_BRANCH=HEAD, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, RELEASE=HEAD, io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-03-21T12:45:24.167 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:45:23 smithi189 podman[59995]: 2024-03-21 12:45:23.801449142 +0000 UTC m=+0.020546337 image pull 19e12293512ff7978c8adb5f72a829060dd47f07bfd6903b6b3178d2ebc29e4e quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93 2024-03-21T12:45:24.167 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:45:23 smithi189 podman[59995]: 2024-03-21 12:45:23.914202155 +0000 UTC m=+0.133299326 container create b64dea20d5859a4494bddf039ee49d005e262cf7b00e80f7a98633b8e665f418 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-5-deactivate, maintainer=Guillaume Abrioux , RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240319, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-03-21T12:45:24.174 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:24 smithi022 ceph-mon[67047]: osdmap e131: 8 total, 7 up, 8 in 2024-03-21T12:45:24.513 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:45:24 smithi189 podman[59995]: 2024-03-21 12:45:24.167345337 +0000 UTC m=+0.386442520 container init b64dea20d5859a4494bddf039ee49d005e262cf7b00e80f7a98633b8e665f418 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-5-deactivate, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, org.label-schema.build-date=20240319, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-03-21T12:45:24.513 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:45:24 smithi189 podman[59995]: 2024-03-21 12:45:24.171903082 +0000 UTC m=+0.391000254 container start b64dea20d5859a4494bddf039ee49d005e262cf7b00e80f7a98633b8e665f418 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-5-deactivate, maintainer=Guillaume Abrioux , ceph=True, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240319, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-21T12:45:24.513 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:45:24 smithi189 podman[59995]: 2024-03-21 12:45:24.267424498 +0000 UTC m=+0.486521678 container attach b64dea20d5859a4494bddf039ee49d005e262cf7b00e80f7a98633b8e665f418 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-5-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, ceph=True, org.label-schema.build-date=20240319, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0) 2024-03-21T12:45:24.513 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:24 smithi189 ceph-mon[53160]: osdmap e131: 8 total, 7 up, 8 in 2024-03-21T12:45:24.786 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:45:24 smithi189 podman[59995]: 2024-03-21 12:45:24.513450863 +0000 UTC m=+0.732548033 container died b64dea20d5859a4494bddf039ee49d005e262cf7b00e80f7a98633b8e665f418 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-5-deactivate, io.buildah.version=1.33.5, org.label-schema.build-date=20240319, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, GIT_BRANCH=HEAD) 2024-03-21T12:45:24.888 INFO:teuthology.orchestra.run.smithi022.stdout: 5 262166 10.08 MB/sec execute 504 sec latency 52.205 ms 2024-03-21T12:45:25.197 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:25 smithi022 ceph-mon[67047]: pgmap v231: 97 pgs: 1 active+undersized, 3 active+undersized+degraded, 5 peering, 8 stale+active+clean, 80 active+clean; 342 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.3 MiB/s wr, 522 op/s; 18/1620 objects degraded (1.111%) 2024-03-21T12:45:25.198 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:25 smithi022 ceph-mon[67047]: Health check failed: Degraded data redundancy: 18/1620 objects degraded (1.111%), 3 pgs degraded (PG_DEGRADED) 2024-03-21T12:45:25.271 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:25 smithi189 ceph-mon[53160]: pgmap v231: 97 pgs: 1 active+undersized, 3 active+undersized+degraded, 5 peering, 8 stale+active+clean, 80 active+clean; 342 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.3 MiB/s wr, 522 op/s; 18/1620 objects degraded (1.111%) 2024-03-21T12:45:25.271 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:25 smithi189 ceph-mon[53160]: Health check failed: Degraded data redundancy: 18/1620 objects degraded (1.111%), 3 pgs degraded (PG_DEGRADED) 2024-03-21T12:45:25.522 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:45:25 smithi189 podman[60021]: 2024-03-21 12:45:25.271598279 +0000 UTC m=+0.747904434 container remove b64dea20d5859a4494bddf039ee49d005e262cf7b00e80f7a98633b8e665f418 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-5-deactivate, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, org.label-schema.build-date=20240319, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-21T12:45:25.522 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:45:25 smithi189 systemd[1]: ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715@osd.5.service: Deactivated successfully. 2024-03-21T12:45:25.522 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:45:25 smithi189 systemd[1]: Stopped Ceph osd.5 for 3fc7f0f2-e77e-11ee-95cd-87774f69a715. 2024-03-21T12:45:25.522 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:45:25 smithi189 systemd[1]: ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715@osd.5.service: Consumed 5min 29.641s CPU time. 2024-03-21T12:45:25.522 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:45:25 smithi189 systemd[1]: Starting Ceph osd.5 for 3fc7f0f2-e77e-11ee-95cd-87774f69a715... 2024-03-21T12:45:25.888 INFO:teuthology.orchestra.run.smithi022.stdout: 5 262500 10.07 MB/sec execute 505 sec latency 38.516 ms 2024-03-21T12:45:26.036 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:45:25 smithi189 podman[60121]: 2024-03-21 12:45:25.669060524 +0000 UTC m=+0.021323588 image pull 19e12293512ff7978c8adb5f72a829060dd47f07bfd6903b6b3178d2ebc29e4e quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93 2024-03-21T12:45:26.036 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:45:25 smithi189 podman[60121]: 2024-03-21 12:45:25.781031249 +0000 UTC m=+0.133294336 container create 22a7ad150b6f5ae8ee5959e974bfd335fbb6d0b9827a93e8ec9f63ff8d883ffe (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-5-activate, GIT_CLEAN=True, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240319, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-03-21T12:45:26.236 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:26 smithi022 ceph-mon[67047]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-03-21T12:45:26.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:26 smithi189 ceph-mon[53160]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-03-21T12:45:26.536 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:45:26 smithi189 podman[60121]: 2024-03-21 12:45:26.062275427 +0000 UTC m=+0.414538483 container init 22a7ad150b6f5ae8ee5959e974bfd335fbb6d0b9827a93e8ec9f63ff8d883ffe (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-5-activate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240319, io.buildah.version=1.33.5, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30) 2024-03-21T12:45:26.536 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:45:26 smithi189 podman[60121]: 2024-03-21 12:45:26.065708421 +0000 UTC m=+0.417971474 container start 22a7ad150b6f5ae8ee5959e974bfd335fbb6d0b9827a93e8ec9f63ff8d883ffe (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-5-activate, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240319, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, GIT_BRANCH=HEAD) 2024-03-21T12:45:26.536 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:45:26 smithi189 podman[60121]: 2024-03-21 12:45:26.16103423 +0000 UTC m=+0.513297287 container attach 22a7ad150b6f5ae8ee5959e974bfd335fbb6d0b9827a93e8ec9f63ff8d883ffe (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-5-activate, io.buildah.version=1.33.5, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240319, maintainer=Guillaume Abrioux , GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30) 2024-03-21T12:45:26.888 INFO:teuthology.orchestra.run.smithi022.stdout: 5 262808 10.07 MB/sec execute 506 sec latency 51.053 ms 2024-03-21T12:45:26.982 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:45:26 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-5-activate[60136]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-03-21T12:45:26.982 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:45:26 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-5-activate[60136]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-03-21T12:45:26.982 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:45:26 smithi189 bash[60121]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-03-21T12:45:26.983 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:45:26 smithi189 bash[60121]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-03-21T12:45:26.983 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:45:26 smithi189 bash[60121]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-03-21T12:45:26.983 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:45:26 smithi189 bash[60121]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-03-21T12:45:26.983 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:45:26 smithi189 bash[60121]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-03-21T12:45:26.983 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:45:26 smithi189 bash[60121]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-03-21T12:45:26.983 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:45:26 smithi189 bash[60121]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-03-21T12:45:26.983 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:45:26 smithi189 bash[60121]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-03-21T12:45:26.983 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:45:26 smithi189 bash[60121]: --> ceph-volume raw activate successful for osd ID: 5 2024-03-21T12:45:26.983 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:45:26 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-5-activate[60136]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-03-21T12:45:26.983 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:45:26 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-5-activate[60136]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-03-21T12:45:26.983 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:45:26 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-5-activate[60136]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-03-21T12:45:26.983 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:45:26 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-5-activate[60136]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-03-21T12:45:26.983 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:45:26 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-5-activate[60136]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-03-21T12:45:26.983 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:45:26 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-5-activate[60136]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-03-21T12:45:26.983 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:45:26 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-5-activate[60136]: --> ceph-volume raw activate successful for osd ID: 5 2024-03-21T12:45:26.984 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:45:26 smithi189 podman[60121]: 2024-03-21 12:45:26.6534713 +0000 UTC m=+1.005734362 container died 22a7ad150b6f5ae8ee5959e974bfd335fbb6d0b9827a93e8ec9f63ff8d883ffe (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-5-activate, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240319, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, ceph=True) 2024-03-21T12:45:27.260 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:27 smithi022 ceph-mon[67047]: pgmap v232: 97 pgs: 3 active+undersized, 6 active+undersized+degraded, 5 peering, 5 stale+active+clean, 78 active+clean; 341 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.9 KiB/s rd, 3.2 MiB/s wr, 294 op/s; 40/1608 objects degraded (2.488%) 2024-03-21T12:45:27.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:27 smithi189 ceph-mon[53160]: pgmap v232: 97 pgs: 3 active+undersized, 6 active+undersized+degraded, 5 peering, 5 stale+active+clean, 78 active+clean; 341 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.9 KiB/s rd, 3.2 MiB/s wr, 294 op/s; 40/1608 objects degraded (2.488%) 2024-03-21T12:45:27.673 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:45:27 smithi189 podman[60269]: 2024-03-21 12:45:27.3071612 +0000 UTC m=+0.646164100 container remove 22a7ad150b6f5ae8ee5959e974bfd335fbb6d0b9827a93e8ec9f63ff8d883ffe (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-5-activate, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240319, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-21T12:45:27.673 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:45:27 smithi189 podman[60331]: 2024-03-21 12:45:27.429382682 +0000 UTC m=+0.018698406 image pull 19e12293512ff7978c8adb5f72a829060dd47f07bfd6903b6b3178d2ebc29e4e quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93 2024-03-21T12:45:27.673 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:45:27 smithi189 podman[60331]: 2024-03-21 12:45:27.550079531 +0000 UTC m=+0.139395249 container create d2c15ba54b4f9587306441a10cb65af7d311a50b1337c97c68b9f8e5b823b288 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-5, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240319, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, io.buildah.version=1.33.5, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-03-21T12:45:27.888 INFO:teuthology.orchestra.run.smithi022.stdout: 5 263253 10.08 MB/sec execute 507 sec latency 54.077 ms 2024-03-21T12:45:28.038 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:45:27 smithi189 podman[60331]: 2024-03-21 12:45:27.794432595 +0000 UTC m=+0.383748315 container init d2c15ba54b4f9587306441a10cb65af7d311a50b1337c97c68b9f8e5b823b288 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-5, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240319, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=) 2024-03-21T12:45:28.038 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:45:27 smithi189 podman[60331]: 2024-03-21 12:45:27.798327547 +0000 UTC m=+0.387643258 container start d2c15ba54b4f9587306441a10cb65af7d311a50b1337c97c68b9f8e5b823b288 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-5, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240319, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, io.buildah.version=1.33.5) 2024-03-21T12:45:28.038 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:45:27 smithi189 ceph-osd[60350]: -- 172.21.15.189:0/1137026007 <== mon.1 v2:172.21.15.189:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x559d2b0b3680 con 0x559d2b07d000 2024-03-21T12:45:28.038 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:45:27 smithi189 bash[60331]: d2c15ba54b4f9587306441a10cb65af7d311a50b1337c97c68b9f8e5b823b288 2024-03-21T12:45:28.038 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:45:27 smithi189 systemd[1]: Started Ceph osd.5 for 3fc7f0f2-e77e-11ee-95cd-87774f69a715. 2024-03-21T12:45:28.786 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:45:28 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-5[60346]: 2024-03-21T12:45:28.349+0000 7f0633c7b7c0 -1 Falling back to public interface 2024-03-21T12:45:28.888 INFO:teuthology.orchestra.run.smithi022.stdout: 5 263636 10.07 MB/sec execute 508 sec latency 44.973 ms 2024-03-21T12:45:29.063 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:28 smithi022 ceph-mon[67047]: pgmap v233: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.4 MiB/s wr, 567 op/s; 175/1572 objects degraded (11.132%) 2024-03-21T12:45:29.063 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:28 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:45:29.063 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:28 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:45:29.063 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:28 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:45:29.063 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:28 smithi022 ceph-mon[67047]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-03-21T12:45:29.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:28 smithi189 ceph-mon[53160]: pgmap v233: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.4 MiB/s wr, 567 op/s; 175/1572 objects degraded (11.132%) 2024-03-21T12:45:29.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:28 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:45:29.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:28 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:45:29.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:28 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:45:29.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:28 smithi189 ceph-mon[53160]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-03-21T12:45:29.888 INFO:teuthology.orchestra.run.smithi022.stdout: 5 263888 10.07 MB/sec execute 509 sec latency 37.402 ms 2024-03-21T12:45:30.018 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:30 smithi189 ceph-mon[53160]: Health check update: Degraded data redundancy: 176/1596 objects degraded (11.028%), 23 pgs degraded (PG_DEGRADED) 2024-03-21T12:45:30.074 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:30 smithi022 ceph-mon[67047]: Health check update: Degraded data redundancy: 176/1596 objects degraded (11.028%), 23 pgs degraded (PG_DEGRADED) 2024-03-21T12:45:30.888 INFO:teuthology.orchestra.run.smithi022.stdout: 5 264240 10.08 MB/sec execute 510 sec latency 62.021 ms 2024-03-21T12:45:31.086 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:31 smithi022 ceph-mon[67047]: pgmap v234: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 342 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.7 KiB/s rd, 4.0 MiB/s wr, 369 op/s; 176/1596 objects degraded (11.028%) 2024-03-21T12:45:31.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:31 smithi189 ceph-mon[53160]: pgmap v234: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 342 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.7 KiB/s rd, 4.0 MiB/s wr, 369 op/s; 176/1596 objects degraded (11.028%) 2024-03-21T12:45:31.888 INFO:teuthology.orchestra.run.smithi022.stdout: 5 264584 10.07 MB/sec execute 511 sec latency 48.506 ms 2024-03-21T12:45:32.888 INFO:teuthology.orchestra.run.smithi022.stdout: 5 264875 10.07 MB/sec execute 512 sec latency 60.128 ms 2024-03-21T12:45:33.124 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:33 smithi022 ceph-mon[67047]: pgmap v235: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 3.6 MiB/s wr, 320 op/s; 180/1617 objects degraded (11.132%) 2024-03-21T12:45:33.124 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:33 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:45:33.124 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:33 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:45:33.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:33 smithi189 ceph-mon[53160]: pgmap v235: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 3.6 MiB/s wr, 320 op/s; 180/1617 objects degraded (11.132%) 2024-03-21T12:45:33.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:33 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:45:33.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:33 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:45:33.888 INFO:teuthology.orchestra.run.smithi022.stdout: 5 265086 10.06 MB/sec execute 513 sec latency 55.383 ms 2024-03-21T12:45:34.406 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:34 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:45:34.407 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:34 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:45:34.407 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:34 smithi189 ceph-mon[53160]: pgmap v236: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 344 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.5 MiB/s wr, 464 op/s; 182/1662 objects degraded (10.951%) 2024-03-21T12:45:34.423 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:34 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:45:34.423 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:34 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:45:34.423 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:34 smithi022 ceph-mon[67047]: pgmap v236: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 344 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.5 MiB/s wr, 464 op/s; 182/1662 objects degraded (10.951%) 2024-03-21T12:45:34.888 INFO:teuthology.orchestra.run.smithi022.stdout: 5 265358 10.05 MB/sec execute 514 sec latency 62.425 ms 2024-03-21T12:45:35.888 INFO:teuthology.orchestra.run.smithi022.stdout: 5 265688 10.05 MB/sec execute 515 sec latency 65.529 ms 2024-03-21T12:45:36.770 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:36 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:45:36.771 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:36 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:45:36.771 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:36 smithi022 ceph-mon[67047]: pgmap v237: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 344 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.1 KiB/s rd, 4.1 MiB/s wr, 418 op/s; 184/1668 objects degraded (11.031%) 2024-03-21T12:45:36.888 INFO:teuthology.orchestra.run.smithi022.stdout: 5 265972 10.04 MB/sec execute 516 sec latency 52.355 ms 2024-03-21T12:45:37.036 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:45:36 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-5[60346]: 2024-03-21T12:45:36.921+0000 7f0633c7b7c0 -1 osd.5 0 read_superblock omap replica is missing. 2024-03-21T12:45:37.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:36 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:45:37.037 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:36 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:45:37.037 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:36 smithi189 ceph-mon[53160]: pgmap v237: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 344 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.1 KiB/s rd, 4.1 MiB/s wr, 418 op/s; 184/1668 objects degraded (11.031%) 2024-03-21T12:45:37.826 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:37 smithi022 ceph-mon[67047]: Health check update: Degraded data redundancy: 184/1668 objects degraded (11.031%), 23 pgs degraded (PG_DEGRADED) 2024-03-21T12:45:37.826 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:37 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:45:37.826 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:37 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:45:37.826 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:37 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:45:37.826 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:37 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:45:37.826 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:37 smithi022 ceph-mon[67047]: pgmap v238: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 343 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 5.2 MiB/s wr, 556 op/s; 185/1659 objects degraded (11.151%) 2024-03-21T12:45:37.826 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:37 smithi022 ceph-mon[67047]: pgmap v239: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 343 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.3 MiB/s wr, 425 op/s; 185/1659 objects degraded (11.151%) 2024-03-21T12:45:37.826 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:37 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:45:37.827 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:37 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:45:37.827 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:37 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:45:37.827 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:37 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:45:37.827 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:37 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:45:37.827 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:37 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:45:37.827 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:37 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-03-21T12:45:37.888 INFO:teuthology.orchestra.run.smithi022.stdout: 5 266219 10.04 MB/sec execute 517 sec latency 73.511 ms 2024-03-21T12:45:38.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:37 smithi189 ceph-mon[53160]: Health check update: Degraded data redundancy: 184/1668 objects degraded (11.031%), 23 pgs degraded (PG_DEGRADED) 2024-03-21T12:45:38.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:37 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:45:38.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:37 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:45:38.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:37 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:45:38.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:37 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:45:38.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:37 smithi189 ceph-mon[53160]: pgmap v238: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 343 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 5.2 MiB/s wr, 556 op/s; 185/1659 objects degraded (11.151%) 2024-03-21T12:45:38.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:37 smithi189 ceph-mon[53160]: pgmap v239: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 343 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.3 MiB/s wr, 425 op/s; 185/1659 objects degraded (11.151%) 2024-03-21T12:45:38.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:37 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:45:38.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:37 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:45:38.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:37 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:45:38.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:37 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:45:38.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:37 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:45:38.037 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:37 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:45:38.037 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:37 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-03-21T12:45:38.786 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:45:38 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-5[60346]: 2024-03-21T12:45:38.289+0000 7f0633c7b7c0 -1 osd.5 129 log_to_monitors true 2024-03-21T12:45:38.839 INFO:teuthology.orchestra.run.smithi022.stdout:true 2024-03-21T12:45:38.888 INFO:teuthology.orchestra.run.smithi022.stdout: 5 266499 10.03 MB/sec execute 518 sec latency 45.831 ms 2024-03-21T12:45:39.357 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:39 smithi022 ceph-mon[67047]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-03-21T12:45:39.358 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:39 smithi022 ceph-mon[67047]: Upgrade: unsafe to stop osd(s) at this time (13 PGs are or would become offline) 2024-03-21T12:45:39.358 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:39 smithi022 ceph-mon[67047]: from='osd.5 [v2:172.21.15.189:6808/2354825655,v1:172.21.15.189:6809/2354825655]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-03-21T12:45:39.358 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:39 smithi022 ceph-mon[67047]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-03-21T12:45:39.535 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:45:39 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-5[60346]: 2024-03-21T12:45:39.312+0000 7f06250d3700 -1 osd.5 129 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-21T12:45:39.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:39 smithi189 ceph-mon[53160]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-03-21T12:45:39.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:39 smithi189 ceph-mon[53160]: Upgrade: unsafe to stop osd(s) at this time (13 PGs are or would become offline) 2024-03-21T12:45:39.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:39 smithi189 ceph-mon[53160]: from='osd.5 [v2:172.21.15.189:6808/2354825655,v1:172.21.15.189:6809/2354825655]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-03-21T12:45:39.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:39 smithi189 ceph-mon[53160]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-03-21T12:45:39.888 INFO:teuthology.orchestra.run.smithi022.stdout: 5 266839 10.03 MB/sec execute 519 sec latency 60.847 ms 2024-03-21T12:45:39.904 INFO:teuthology.orchestra.run.smithi022.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-21T12:45:39.904 INFO:teuthology.orchestra.run.smithi022.stdout:alertmanager.smithi022 smithi022 *:9093,9094 running (8m) 82s ago 17m 24.1M - 0.23.0 ba2b418f427c 14eb2a3ca318 2024-03-21T12:45:39.905 INFO:teuthology.orchestra.run.smithi022.stdout:crash.smithi022 smithi022 running (4m) 82s ago 16m 6660k - 19.0.0-2267-g960fbbe4 19e12293512f 356b2ddb609a 2024-03-21T12:45:39.905 INFO:teuthology.orchestra.run.smithi022.stdout:crash.smithi189 smithi189 running (4m) 7s ago 16m 6660k - 19.0.0-2267-g960fbbe4 19e12293512f 90ee837f0500 2024-03-21T12:45:39.905 INFO:teuthology.orchestra.run.smithi022.stdout:grafana.smithi022 smithi022 *:3000 error 82s ago 16m - - 2024-03-21T12:45:39.905 INFO:teuthology.orchestra.run.smithi022.stdout:mds.foofs.smithi022.bcnxho smithi022 running (11m) 82s ago 11m 38.3M - 17.2.0 e1d6a67b021e db26b67b8ee7 2024-03-21T12:45:39.905 INFO:teuthology.orchestra.run.smithi022.stdout:mds.foofs.smithi189.lluvko smithi189 running (11m) 7s ago 11m 17.7M - 17.2.0 e1d6a67b021e cf153aff1387 2024-03-21T12:45:39.905 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.smithi022.fektzs smithi022 *:8443,9283,8765 running (7m) 82s ago 18m 506M - 19.0.0-2267-g960fbbe4 19e12293512f d57cebd306b3 2024-03-21T12:45:39.905 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.smithi189.cnxzvu smithi189 *:8443,9283,8765 running (6m) 7s ago 16m 452M - 19.0.0-2267-g960fbbe4 19e12293512f 41663955c56f 2024-03-21T12:45:39.905 INFO:teuthology.orchestra.run.smithi022.stdout:mon.smithi022 smithi022 running (5m) 82s ago 18m 50.6M 2048M 19.0.0-2267-g960fbbe4 19e12293512f c620c9071b2c 2024-03-21T12:45:39.905 INFO:teuthology.orchestra.run.smithi022.stdout:mon.smithi189 smithi189 running (5m) 7s ago 16m 43.3M 2048M 19.0.0-2267-g960fbbe4 19e12293512f b353af3d8f3b 2024-03-21T12:45:39.905 INFO:teuthology.orchestra.run.smithi022.stdout:nfs.foo.0.0.smithi022.ynhbbq smithi022 *:2049 running (10m) 82s ago 10m 277M - 4.0 e1d6a67b021e 33e48e64c60b 2024-03-21T12:45:39.905 INFO:teuthology.orchestra.run.smithi022.stdout:nfs.foo.1.0.smithi189.yzsqml smithi189 *:2049 running (10m) 7s ago 10m 86.0M - 4.0 e1d6a67b021e 48fab2cbdde2 2024-03-21T12:45:39.905 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.smithi022 smithi022 *:9100 running (7m) 82s ago 16m 18.2M - 1.3.1 1dbe0e931976 33287f803f4f 2024-03-21T12:45:39.905 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.smithi189 smithi189 *:9100 running (7m) 7s ago 16m 17.6M - 1.3.1 1dbe0e931976 b51d872e438a 2024-03-21T12:45:39.905 INFO:teuthology.orchestra.run.smithi022.stdout:osd.0 smithi022 running (3m) 82s ago 15m 336M 1937M 19.0.0-2267-g960fbbe4 19e12293512f 23a85645eeea 2024-03-21T12:45:39.906 INFO:teuthology.orchestra.run.smithi022.stdout:osd.1 smithi022 running (2m) 82s ago 15m 286M 1937M 19.0.0-2267-g960fbbe4 19e12293512f 0f90e70bbf0a 2024-03-21T12:45:39.906 INFO:teuthology.orchestra.run.smithi022.stdout:osd.2 smithi022 running (2m) 82s ago 14m 214M 1937M 19.0.0-2267-g960fbbe4 19e12293512f 78a229f593f8 2024-03-21T12:45:39.906 INFO:teuthology.orchestra.run.smithi022.stdout:osd.3 smithi022 running (91s) 82s ago 14m 12.5M 1937M 19.0.0-2267-g960fbbe4 19e12293512f 61a63b67d9af 2024-03-21T12:45:39.906 INFO:teuthology.orchestra.run.smithi022.stdout:osd.4 smithi189 running (50s) 7s ago 13m 270M 2704M 19.0.0-2267-g960fbbe4 19e12293512f 382755cee998 2024-03-21T12:45:39.906 INFO:teuthology.orchestra.run.smithi022.stdout:osd.5 smithi189 running (12s) 7s ago 13m 12.1M 2704M 19.0.0-2267-g960fbbe4 19e12293512f d2c15ba54b4f 2024-03-21T12:45:39.906 INFO:teuthology.orchestra.run.smithi022.stdout:osd.6 smithi189 running (13m) 7s ago 13m 554M 2704M 17.2.0 e1d6a67b021e 0c192fcfd65b 2024-03-21T12:45:39.906 INFO:teuthology.orchestra.run.smithi022.stdout:osd.7 smithi189 running (12m) 7s ago 12m 606M 2704M 17.2.0 e1d6a67b021e 3d0723e1f863 2024-03-21T12:45:39.906 INFO:teuthology.orchestra.run.smithi022.stdout:prometheus.smithi022 smithi022 *:9095 running (6m) 82s ago 15m 73.9M - 2.33.4 514e6a882f6e 8171bbe41d2c 2024-03-21T12:45:40.364 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:40 smithi022 ceph-mon[67047]: from='client.34388 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:45:40.365 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:40 smithi022 ceph-mon[67047]: pgmap v240: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 333 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.6 MiB/s wr, 479 op/s; 174/1587 objects degraded (10.964%) 2024-03-21T12:45:40.365 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:40 smithi022 ceph-mon[67047]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-03-21T12:45:40.365 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:40 smithi022 ceph-mon[67047]: osdmap e132: 8 total, 7 up, 8 in 2024-03-21T12:45:40.365 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:40 smithi022 ceph-mon[67047]: from='osd.5 [v2:172.21.15.189:6808/2354825655,v1:172.21.15.189:6809/2354825655]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-03-21T12:45:40.365 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:40 smithi022 ceph-mon[67047]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-03-21T12:45:40.524 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-03-21T12:45:40.524 INFO:teuthology.orchestra.run.smithi022.stdout: "mon": { 2024-03-21T12:45:40.524 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 2 2024-03-21T12:45:40.524 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:45:40.524 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr": { 2024-03-21T12:45:40.525 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 2 2024-03-21T12:45:40.525 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:45:40.525 INFO:teuthology.orchestra.run.smithi022.stdout: "osd": { 2024-03-21T12:45:40.525 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-03-21T12:45:40.525 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 6 2024-03-21T12:45:40.525 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:45:40.525 INFO:teuthology.orchestra.run.smithi022.stdout: "mds": { 2024-03-21T12:45:40.525 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-21T12:45:40.525 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:45:40.525 INFO:teuthology.orchestra.run.smithi022.stdout: "overall": { 2024-03-21T12:45:40.525 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-03-21T12:45:40.525 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 10 2024-03-21T12:45:40.525 INFO:teuthology.orchestra.run.smithi022.stdout: } 2024-03-21T12:45:40.525 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-03-21T12:45:40.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:40 smithi189 ceph-mon[53160]: from='client.34388 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:45:40.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:40 smithi189 ceph-mon[53160]: pgmap v240: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 333 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.6 MiB/s wr, 479 op/s; 174/1587 objects degraded (10.964%) 2024-03-21T12:45:40.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:40 smithi189 ceph-mon[53160]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-03-21T12:45:40.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:40 smithi189 ceph-mon[53160]: osdmap e132: 8 total, 7 up, 8 in 2024-03-21T12:45:40.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:40 smithi189 ceph-mon[53160]: from='osd.5 [v2:172.21.15.189:6808/2354825655,v1:172.21.15.189:6809/2354825655]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-03-21T12:45:40.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:40 smithi189 ceph-mon[53160]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-03-21T12:45:40.888 INFO:teuthology.orchestra.run.smithi022.stdout: 5 267082 10.02 MB/sec execute 520 sec latency 551.753 ms 2024-03-21T12:45:40.916 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-03-21T12:45:40.917 INFO:teuthology.orchestra.run.smithi022.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93", 2024-03-21T12:45:40.917 INFO:teuthology.orchestra.run.smithi022.stdout: "in_progress": true, 2024-03-21T12:45:40.917 INFO:teuthology.orchestra.run.smithi022.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-21T12:45:40.917 INFO:teuthology.orchestra.run.smithi022.stdout: "services_complete": [ 2024-03-21T12:45:40.917 INFO:teuthology.orchestra.run.smithi022.stdout: "mon", 2024-03-21T12:45:40.917 INFO:teuthology.orchestra.run.smithi022.stdout: "crash", 2024-03-21T12:45:40.917 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr" 2024-03-21T12:45:40.917 INFO:teuthology.orchestra.run.smithi022.stdout: ], 2024-03-21T12:45:40.917 INFO:teuthology.orchestra.run.smithi022.stdout: "progress": "12/23 daemons upgraded", 2024-03-21T12:45:40.917 INFO:teuthology.orchestra.run.smithi022.stdout: "message": "Currently upgrading osd daemons", 2024-03-21T12:45:40.917 INFO:teuthology.orchestra.run.smithi022.stdout: "is_paused": false 2024-03-21T12:45:40.917 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-03-21T12:45:41.337 INFO:teuthology.orchestra.run.smithi022.stdout:HEALTH_WARN 1 failed cephadm daemon(s); Degraded data redundancy: 183/1599 objects degraded (11.445%), 23 pgs degraded 2024-03-21T12:45:41.338 INFO:teuthology.orchestra.run.smithi022.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-21T12:45:41.338 INFO:teuthology.orchestra.run.smithi022.stdout: daemon grafana.smithi022 on smithi022 is in error state 2024-03-21T12:45:41.338 INFO:teuthology.orchestra.run.smithi022.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 183/1599 objects degraded (11.445%), 23 pgs degraded 2024-03-21T12:45:41.338 INFO:teuthology.orchestra.run.smithi022.stdout: pg 2.2 is active+undersized+degraded, acting [1,6] 2024-03-21T12:45:41.338 INFO:teuthology.orchestra.run.smithi022.stdout: pg 2.3 is active+undersized+degraded, acting [2,7] 2024-03-21T12:45:41.338 INFO:teuthology.orchestra.run.smithi022.stdout: pg 2.b is active+undersized+degraded, acting [7,4] 2024-03-21T12:45:41.338 INFO:teuthology.orchestra.run.smithi022.stdout: pg 2.c is active+undersized+degraded, acting [2,0] 2024-03-21T12:45:41.338 INFO:teuthology.orchestra.run.smithi022.stdout: pg 2.12 is active+undersized+degraded, acting [3,7] 2024-03-21T12:45:41.338 INFO:teuthology.orchestra.run.smithi022.stdout: pg 2.14 is active+undersized+degraded, acting [6,3] 2024-03-21T12:45:41.338 INFO:teuthology.orchestra.run.smithi022.stdout: pg 2.15 is active+undersized+degraded, acting [1,0] 2024-03-21T12:45:41.338 INFO:teuthology.orchestra.run.smithi022.stdout: pg 2.16 is active+undersized+degraded, acting [6,2] 2024-03-21T12:45:41.338 INFO:teuthology.orchestra.run.smithi022.stdout: pg 2.17 is active+undersized+degraded, acting [6,2] 2024-03-21T12:45:41.338 INFO:teuthology.orchestra.run.smithi022.stdout: pg 2.18 is active+undersized+degraded, acting [4,7] 2024-03-21T12:45:41.338 INFO:teuthology.orchestra.run.smithi022.stdout: pg 2.1c is active+undersized+degraded, acting [7,2] 2024-03-21T12:45:41.338 INFO:teuthology.orchestra.run.smithi022.stdout: pg 2.1e is active+undersized+degraded, acting [3,0] 2024-03-21T12:45:41.339 INFO:teuthology.orchestra.run.smithi022.stdout: pg 3.2 is active+undersized+degraded, acting [3,6] 2024-03-21T12:45:41.339 INFO:teuthology.orchestra.run.smithi022.stdout: pg 3.4 is active+undersized+degraded, acting [1,2] 2024-03-21T12:45:41.339 INFO:teuthology.orchestra.run.smithi022.stdout: pg 3.5 is active+undersized+degraded, acting [3,2] 2024-03-21T12:45:41.339 INFO:teuthology.orchestra.run.smithi022.stdout: pg 3.c is active+undersized+degraded, acting [3,6] 2024-03-21T12:45:41.339 INFO:teuthology.orchestra.run.smithi022.stdout: pg 3.d is active+undersized+degraded, acting [7,6] 2024-03-21T12:45:41.339 INFO:teuthology.orchestra.run.smithi022.stdout: pg 3.10 is active+undersized+degraded, acting [6,0] 2024-03-21T12:45:41.339 INFO:teuthology.orchestra.run.smithi022.stdout: pg 3.16 is active+undersized+degraded, acting [7,1] 2024-03-21T12:45:41.339 INFO:teuthology.orchestra.run.smithi022.stdout: pg 3.17 is active+undersized+degraded, acting [0,3] 2024-03-21T12:45:41.339 INFO:teuthology.orchestra.run.smithi022.stdout: pg 3.1c is active+undersized+degraded, acting [4,1] 2024-03-21T12:45:41.339 INFO:teuthology.orchestra.run.smithi022.stdout: pg 3.1d is active+undersized+degraded, acting [4,6] 2024-03-21T12:45:41.339 INFO:teuthology.orchestra.run.smithi022.stdout: pg 3.1f is active+undersized+degraded, acting [0,2] 2024-03-21T12:45:41.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:41 smithi022 ceph-mon[67047]: from='client.44285 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:45:41.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:41 smithi022 ceph-mon[67047]: from='client.34396 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:45:41.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:41 smithi022 ceph-mon[67047]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-21T12:45:41.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:41 smithi022 ceph-mon[67047]: osd.5 [v2:172.21.15.189:6808/2354825655,v1:172.21.15.189:6809/2354825655] boot 2024-03-21T12:45:41.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:41 smithi022 ceph-mon[67047]: osdmap e133: 8 total, 8 up, 8 in 2024-03-21T12:45:41.595 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:41 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-21T12:45:41.595 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:41 smithi022 ceph-mon[67047]: from='client.? 172.21.15.22:0/513217063' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:45:41.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:41 smithi189 ceph-mon[53160]: from='client.44285 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:45:41.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:41 smithi189 ceph-mon[53160]: from='client.34396 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:45:41.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:41 smithi189 ceph-mon[53160]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-21T12:45:41.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:41 smithi189 ceph-mon[53160]: osd.5 [v2:172.21.15.189:6808/2354825655,v1:172.21.15.189:6809/2354825655] boot 2024-03-21T12:45:41.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:41 smithi189 ceph-mon[53160]: osdmap e133: 8 total, 8 up, 8 in 2024-03-21T12:45:41.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:41 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-21T12:45:41.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:41 smithi189 ceph-mon[53160]: from='client.? 172.21.15.22:0/513217063' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:45:41.888 INFO:teuthology.orchestra.run.smithi022.stdout: 5 267082 10.00 MB/sec execute 521 sec latency 1551.800 ms 2024-03-21T12:45:42.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:42 smithi022 ceph-mon[67047]: from='client.34404 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:45:42.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:42 smithi022 ceph-mon[67047]: pgmap v243: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 334 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 3.8 MiB/s wr, 414 op/s; 183/1599 objects degraded (11.445%) 2024-03-21T12:45:42.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:42 smithi022 ceph-mon[67047]: osdmap e134: 8 total, 8 up, 8 in 2024-03-21T12:45:42.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:42 smithi022 ceph-mon[67047]: from='client.? 172.21.15.22:0/3411932001' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-21T12:45:42.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:42 smithi189 ceph-mon[53160]: from='client.34404 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:45:42.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:42 smithi189 ceph-mon[53160]: pgmap v243: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 334 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 3.8 MiB/s wr, 414 op/s; 183/1599 objects degraded (11.445%) 2024-03-21T12:45:42.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:42 smithi189 ceph-mon[53160]: osdmap e134: 8 total, 8 up, 8 in 2024-03-21T12:45:42.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:42 smithi189 ceph-mon[53160]: from='client.? 172.21.15.22:0/3411932001' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-21T12:45:42.888 INFO:teuthology.orchestra.run.smithi022.stdout: 5 267082 9.98 MB/sec execute 522 sec latency 2551.846 ms 2024-03-21T12:45:43.423 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:43 smithi022 ceph-mon[67047]: osdmap e135: 8 total, 8 up, 8 in 2024-03-21T12:45:43.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:43 smithi189 ceph-mon[53160]: osdmap e135: 8 total, 8 up, 8 in 2024-03-21T12:45:43.888 INFO:teuthology.orchestra.run.smithi022.stdout: 5 267232 9.97 MB/sec execute 523 sec latency 2983.795 ms 2024-03-21T12:45:44.581 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:44 smithi022 ceph-mon[67047]: pgmap v246: 97 pgs: 2 activating+undersized+degraded+remapped, 4 remapped+peering, 5 active+undersized, 9 active+undersized+degraded, 77 active+clean; 329 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 2.2 MiB/s wr, 124 op/s; 110/1578 objects degraded (6.971%); 102 KiB/s, 9 objects/s recovering 2024-03-21T12:45:44.582 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:44 smithi022 ceph-mon[67047]: Health check update: Degraded data redundancy: 110/1578 objects degraded (6.971%), 11 pgs degraded (PG_DEGRADED) 2024-03-21T12:45:44.582 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:44 smithi022 ceph-mon[67047]: osdmap e136: 8 total, 8 up, 8 in 2024-03-21T12:45:44.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:44 smithi189 ceph-mon[53160]: pgmap v246: 97 pgs: 2 activating+undersized+degraded+remapped, 4 remapped+peering, 5 active+undersized, 9 active+undersized+degraded, 77 active+clean; 329 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 2.2 MiB/s wr, 124 op/s; 110/1578 objects degraded (6.971%); 102 KiB/s, 9 objects/s recovering 2024-03-21T12:45:44.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:44 smithi189 ceph-mon[53160]: Health check update: Degraded data redundancy: 110/1578 objects degraded (6.971%), 11 pgs degraded (PG_DEGRADED) 2024-03-21T12:45:44.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:44 smithi189 ceph-mon[53160]: osdmap e136: 8 total, 8 up, 8 in 2024-03-21T12:45:44.888 INFO:teuthology.orchestra.run.smithi022.stdout: 5 267494 9.97 MB/sec execute 524 sec latency 140.589 ms 2024-03-21T12:45:45.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:45 smithi022 ceph-mon[67047]: osdmap e137: 8 total, 8 up, 8 in 2024-03-21T12:45:45.846 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:45 smithi022 ceph-mon[67047]: pgmap v249: 97 pgs: 2 peering, 2 activating+undersized+degraded+remapped, 2 remapped+peering, 1 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 331 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 2.9 MiB/s wr, 285 op/s; 67/1569 objects degraded (4.270%); 467 KiB/s, 24 objects/s recovering 2024-03-21T12:45:45.888 INFO:teuthology.orchestra.run.smithi022.stdout: 5 267784 9.97 MB/sec execute 525 sec latency 862.929 ms 2024-03-21T12:45:46.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:45 smithi189 ceph-mon[53160]: osdmap e137: 8 total, 8 up, 8 in 2024-03-21T12:45:46.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:45 smithi189 ceph-mon[53160]: pgmap v249: 97 pgs: 2 peering, 2 activating+undersized+degraded+remapped, 2 remapped+peering, 1 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 331 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 2.9 MiB/s wr, 285 op/s; 67/1569 objects degraded (4.270%); 467 KiB/s, 24 objects/s recovering 2024-03-21T12:45:46.862 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:46 smithi022 ceph-mon[67047]: osdmap e138: 8 total, 8 up, 8 in 2024-03-21T12:45:46.888 INFO:teuthology.orchestra.run.smithi022.stdout: 5 268096 9.96 MB/sec execute 526 sec latency 973.932 ms 2024-03-21T12:45:47.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:46 smithi189 ceph-mon[53160]: osdmap e138: 8 total, 8 up, 8 in 2024-03-21T12:45:47.874 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:47 smithi022 ceph-mon[67047]: osdmap e139: 8 total, 8 up, 8 in 2024-03-21T12:45:47.875 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:47 smithi022 ceph-mon[67047]: pgmap v252: 97 pgs: 3 peering, 2 activating+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 337 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 2.7 KiB/s rd, 4.0 MiB/s wr, 483 op/s; 67/1608 objects degraded (4.167%); 365 KiB/s, 27 objects/s recovering 2024-03-21T12:45:47.888 INFO:teuthology.orchestra.run.smithi022.stdout: 5 268369 9.95 MB/sec execute 527 sec latency 658.219 ms 2024-03-21T12:45:48.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:47 smithi189 ceph-mon[53160]: osdmap e139: 8 total, 8 up, 8 in 2024-03-21T12:45:48.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:47 smithi189 ceph-mon[53160]: pgmap v252: 97 pgs: 3 peering, 2 activating+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 337 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 2.7 KiB/s rd, 4.0 MiB/s wr, 483 op/s; 67/1608 objects degraded (4.167%); 365 KiB/s, 27 objects/s recovering 2024-03-21T12:45:48.889 INFO:teuthology.orchestra.run.smithi022.stdout: 5 268695 9.96 MB/sec execute 528 sec latency 62.871 ms 2024-03-21T12:45:48.909 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:48 smithi022 ceph-mon[67047]: osdmap e140: 8 total, 8 up, 8 in 2024-03-21T12:45:48.910 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:48 smithi022 ceph-mon[67047]: Health check update: Degraded data redundancy: 67/1608 objects degraded (4.167%), 3 pgs degraded (PG_DEGRADED) 2024-03-21T12:45:49.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:48 smithi189 ceph-mon[53160]: osdmap e140: 8 total, 8 up, 8 in 2024-03-21T12:45:49.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:48 smithi189 ceph-mon[53160]: Health check update: Degraded data redundancy: 67/1608 objects degraded (4.167%), 3 pgs degraded (PG_DEGRADED) 2024-03-21T12:45:49.889 INFO:teuthology.orchestra.run.smithi022.stdout: 5 269027 9.96 MB/sec execute 529 sec latency 52.567 ms 2024-03-21T12:45:49.921 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:49 smithi022 ceph-mon[67047]: pgmap v254: 97 pgs: 3 peering, 1 activating+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 338 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 2.9 KiB/s rd, 3.6 MiB/s wr, 310 op/s; 47/1620 objects degraded (2.901%); 173 KiB/s, 15 objects/s recovering 2024-03-21T12:45:50.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:49 smithi189 ceph-mon[53160]: pgmap v254: 97 pgs: 3 peering, 1 activating+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 338 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 2.9 KiB/s rd, 3.6 MiB/s wr, 310 op/s; 47/1620 objects degraded (2.901%); 173 KiB/s, 15 objects/s recovering 2024-03-21T12:45:50.889 INFO:teuthology.orchestra.run.smithi022.stdout: 5 269377 9.95 MB/sec execute 530 sec latency 37.124 ms 2024-03-21T12:45:51.889 INFO:teuthology.orchestra.run.smithi022.stdout: 5 269682 9.95 MB/sec execute 531 sec latency 34.789 ms 2024-03-21T12:45:51.945 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:51 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:45:51.945 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:51 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:45:51.945 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:51 smithi022 ceph-mon[67047]: pgmap v255: 97 pgs: 1 peering, 96 active+clean; 334 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 2.2 KiB/s rd, 3.1 MiB/s wr, 261 op/s; 956 KiB/s, 22 objects/s recovering 2024-03-21T12:45:52.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:51 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:45:52.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:51 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:45:52.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:51 smithi189 ceph-mon[53160]: pgmap v255: 97 pgs: 1 peering, 96 active+clean; 334 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 2.2 KiB/s rd, 3.1 MiB/s wr, 261 op/s; 956 KiB/s, 22 objects/s recovering 2024-03-21T12:45:52.889 INFO:teuthology.orchestra.run.smithi022.stdout: 5 270018 9.95 MB/sec execute 532 sec latency 39.290 ms 2024-03-21T12:45:52.975 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:52 smithi022 ceph-mon[67047]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 47/1620 objects degraded (2.901%), 2 pgs degraded) 2024-03-21T12:45:52.976 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:52 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-03-21T12:45:53.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:52 smithi189 ceph-mon[53160]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 47/1620 objects degraded (2.901%), 2 pgs degraded) 2024-03-21T12:45:53.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:52 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-03-21T12:45:53.754 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:53 smithi022 ceph-mon[67047]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-03-21T12:45:53.754 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:53 smithi022 ceph-mon[67047]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-03-21T12:45:53.755 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:53 smithi022 ceph-mon[67047]: pgmap v256: 97 pgs: 97 active+clean; 338 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 5.3 MiB/s wr, 542 op/s; 766 KiB/s, 18 objects/s recovering 2024-03-21T12:45:53.889 INFO:teuthology.orchestra.run.smithi022.stdout: 5 270367 9.95 MB/sec execute 533 sec latency 60.679 ms 2024-03-21T12:45:54.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:53 smithi189 ceph-mon[53160]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-03-21T12:45:54.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:53 smithi189 ceph-mon[53160]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-03-21T12:45:54.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:53 smithi189 ceph-mon[53160]: pgmap v256: 97 pgs: 97 active+clean; 338 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 5.3 MiB/s wr, 542 op/s; 766 KiB/s, 18 objects/s recovering 2024-03-21T12:45:54.889 INFO:teuthology.orchestra.run.smithi022.stdout: 5 270671 9.95 MB/sec execute 534 sec latency 64.175 ms 2024-03-21T12:45:55.889 INFO:teuthology.orchestra.run.smithi022.stdout: 5 270948 9.94 MB/sec execute 535 sec latency 51.236 ms 2024-03-21T12:45:56.577 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:56 smithi022 ceph-mon[67047]: pgmap v257: 97 pgs: 97 active+clean; 334 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4.1 KiB/s rd, 4.2 MiB/s wr, 383 op/s; 676 KiB/s, 10 objects/s recovering 2024-03-21T12:45:56.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:56 smithi189 ceph-mon[53160]: pgmap v257: 97 pgs: 97 active+clean; 334 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4.1 KiB/s rd, 4.2 MiB/s wr, 383 op/s; 676 KiB/s, 10 objects/s recovering 2024-03-21T12:45:56.889 INFO:teuthology.orchestra.run.smithi022.stdout: 5 271257 9.93 MB/sec execute 536 sec latency 67.798 ms 2024-03-21T12:45:57.889 INFO:teuthology.orchestra.run.smithi022.stdout: 5 271603 9.93 MB/sec execute 537 sec latency 54.344 ms 2024-03-21T12:45:58.352 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:58 smithi022 ceph-mon[67047]: pgmap v258: 97 pgs: 97 active+clean; 335 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.5 MiB/s wr, 434 op/s; 574 KiB/s, 8 objects/s recovering 2024-03-21T12:45:58.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:58 smithi189 ceph-mon[53160]: pgmap v258: 97 pgs: 97 active+clean; 335 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.5 MiB/s wr, 434 op/s; 574 KiB/s, 8 objects/s recovering 2024-03-21T12:45:58.889 INFO:teuthology.orchestra.run.smithi022.stdout: 5 271984 9.93 MB/sec execute 538 sec latency 43.374 ms 2024-03-21T12:45:59.886 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:45:59 smithi022 ceph-mon[67047]: pgmap v259: 97 pgs: 97 active+clean; 337 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.5 MiB/s wr, 432 op/s; 500 KiB/s, 7 objects/s recovering 2024-03-21T12:45:59.889 INFO:teuthology.orchestra.run.smithi022.stdout: 5 272436 9.94 MB/sec execute 539 sec latency 59.849 ms 2024-03-21T12:46:00.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:45:59 smithi189 ceph-mon[53160]: pgmap v259: 97 pgs: 97 active+clean; 337 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.5 MiB/s wr, 432 op/s; 500 KiB/s, 7 objects/s recovering 2024-03-21T12:46:00.889 INFO:teuthology.orchestra.run.smithi022.stdout: 5 272836 9.94 MB/sec execute 540 sec latency 46.326 ms 2024-03-21T12:46:01.889 INFO:teuthology.orchestra.run.smithi022.stdout: 5 273058 9.93 MB/sec execute 541 sec latency 49.261 ms 2024-03-21T12:46:01.930 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:01 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:46:01.930 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:01 smithi022 ceph-mon[67047]: pgmap v260: 97 pgs: 97 active+clean; 338 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.6 MiB/s wr, 488 op/s; 413 KiB/s, 5 objects/s recovering 2024-03-21T12:46:02.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:01 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:46:02.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:01 smithi189 ceph-mon[53160]: pgmap v260: 97 pgs: 97 active+clean; 338 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.6 MiB/s wr, 488 op/s; 413 KiB/s, 5 objects/s recovering 2024-03-21T12:46:02.889 INFO:teuthology.orchestra.run.smithi022.stdout: 5 273312 9.93 MB/sec execute 542 sec latency 43.935 ms 2024-03-21T12:46:03.889 INFO:teuthology.orchestra.run.smithi022.stdout: 5 273681 9.93 MB/sec execute 543 sec latency 44.791 ms 2024-03-21T12:46:04.487 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:04 smithi022 ceph-mon[67047]: pgmap v261: 97 pgs: 97 active+clean; 337 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 5.9 MiB/s wr, 604 op/s 2024-03-21T12:46:04.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:04 smithi189 ceph-mon[53160]: pgmap v261: 97 pgs: 97 active+clean; 337 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 5.9 MiB/s wr, 604 op/s 2024-03-21T12:46:04.889 INFO:teuthology.orchestra.run.smithi022.stdout: 5 274113 9.93 MB/sec execute 544 sec latency 39.933 ms 2024-03-21T12:46:05.889 INFO:teuthology.orchestra.run.smithi022.stdout: 5 274401 9.93 MB/sec execute 545 sec latency 54.871 ms 2024-03-21T12:46:06.521 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:06 smithi022 ceph-mon[67047]: pgmap v262: 97 pgs: 97 active+clean; 333 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.5 MiB/s wr, 443 op/s 2024-03-21T12:46:06.522 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:06 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:46:06.522 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:06 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:46:06.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:06 smithi189 ceph-mon[53160]: pgmap v262: 97 pgs: 97 active+clean; 333 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.5 MiB/s wr, 443 op/s 2024-03-21T12:46:06.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:06 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:46:06.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:06 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:46:06.889 INFO:teuthology.orchestra.run.smithi022.stdout: 5 274756 9.93 MB/sec execute 546 sec latency 59.456 ms 2024-03-21T12:46:07.889 INFO:teuthology.orchestra.run.smithi022.stdout: 5 275089 9.92 MB/sec execute 547 sec latency 53.325 ms 2024-03-21T12:46:08.473 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:08 smithi189 ceph-mon[53160]: pgmap v263: 97 pgs: 97 active+clean; 333 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.0 MiB/s wr, 452 op/s 2024-03-21T12:46:08.473 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:08 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-03-21T12:46:08.554 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:08 smithi022 ceph-mon[67047]: pgmap v263: 97 pgs: 97 active+clean; 333 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.0 MiB/s wr, 452 op/s 2024-03-21T12:46:08.555 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:08 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-03-21T12:46:08.889 INFO:teuthology.orchestra.run.smithi022.stdout: 5 275385 9.92 MB/sec execute 548 sec latency 95.349 ms 2024-03-21T12:46:09.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:09 smithi189 ceph-mon[53160]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-03-21T12:46:09.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:09 smithi189 ceph-mon[53160]: Upgrade: osd.6 is safe to restart 2024-03-21T12:46:09.571 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:09 smithi022 ceph-mon[67047]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-03-21T12:46:09.571 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:09 smithi022 ceph-mon[67047]: Upgrade: osd.6 is safe to restart 2024-03-21T12:46:09.889 INFO:teuthology.orchestra.run.smithi022.stdout: 5 275712 9.92 MB/sec execute 549 sec latency 46.378 ms 2024-03-21T12:46:10.359 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:10 smithi022 ceph-mon[67047]: pgmap v264: 97 pgs: 97 active+clean; 341 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 5.0 MiB/s wr, 513 op/s 2024-03-21T12:46:10.359 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:10 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:46:10.360 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:10 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-03-21T12:46:10.360 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:10 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:46:10.623 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:10 smithi189 ceph-mon[53160]: pgmap v264: 97 pgs: 97 active+clean; 341 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 5.0 MiB/s wr, 513 op/s 2024-03-21T12:46:10.623 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:10 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:46:10.624 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:10 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-03-21T12:46:10.624 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:10 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:46:10.889 INFO:teuthology.orchestra.run.smithi022.stdout: 5 276053 9.92 MB/sec execute 550 sec latency 50.813 ms 2024-03-21T12:46:11.365 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:11 smithi022 ceph-mon[67047]: Upgrade: Updating osd.6 2024-03-21T12:46:11.366 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:11 smithi022 ceph-mon[67047]: Deploying daemon osd.6 on smithi189 2024-03-21T12:46:11.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:11 smithi189 ceph-mon[53160]: Upgrade: Updating osd.6 2024-03-21T12:46:11.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:11 smithi189 ceph-mon[53160]: Deploying daemon osd.6 on smithi189 2024-03-21T12:46:11.841 INFO:teuthology.orchestra.run.smithi022.stdout:true 2024-03-21T12:46:11.889 INFO:teuthology.orchestra.run.smithi022.stdout: 5 276412 9.92 MB/sec execute 551 sec latency 54.046 ms 2024-03-21T12:46:12.425 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:12 smithi022 ceph-mon[67047]: pgmap v265: 97 pgs: 97 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.8 MiB/s wr, 481 op/s 2024-03-21T12:46:12.745 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:12 smithi189 ceph-mon[53160]: pgmap v265: 97 pgs: 97 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.8 MiB/s wr, 481 op/s 2024-03-21T12:46:12.757 INFO:teuthology.orchestra.run.smithi022.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-21T12:46:12.758 INFO:teuthology.orchestra.run.smithi022.stdout:alertmanager.smithi022 smithi022 *:9093,9094 running (8m) 115s ago 17m 24.1M - 0.23.0 ba2b418f427c 14eb2a3ca318 2024-03-21T12:46:12.758 INFO:teuthology.orchestra.run.smithi022.stdout:crash.smithi022 smithi022 running (4m) 115s ago 17m 6660k - 19.0.0-2267-g960fbbe4 19e12293512f 356b2ddb609a 2024-03-21T12:46:12.758 INFO:teuthology.orchestra.run.smithi022.stdout:crash.smithi189 smithi189 running (4m) 40s ago 16m 6660k - 19.0.0-2267-g960fbbe4 19e12293512f 90ee837f0500 2024-03-21T12:46:12.758 INFO:teuthology.orchestra.run.smithi022.stdout:grafana.smithi022 smithi022 *:3000 error 115s ago 17m - - 2024-03-21T12:46:12.758 INFO:teuthology.orchestra.run.smithi022.stdout:mds.foofs.smithi022.bcnxho smithi022 running (11m) 115s ago 11m 38.3M - 17.2.0 e1d6a67b021e db26b67b8ee7 2024-03-21T12:46:12.758 INFO:teuthology.orchestra.run.smithi022.stdout:mds.foofs.smithi189.lluvko smithi189 running (11m) 40s ago 11m 17.7M - 17.2.0 e1d6a67b021e cf153aff1387 2024-03-21T12:46:12.758 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.smithi022.fektzs smithi022 *:8443,9283,8765 running (7m) 115s ago 18m 506M - 19.0.0-2267-g960fbbe4 19e12293512f d57cebd306b3 2024-03-21T12:46:12.758 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.smithi189.cnxzvu smithi189 *:8443,9283,8765 running (6m) 40s ago 16m 452M - 19.0.0-2267-g960fbbe4 19e12293512f 41663955c56f 2024-03-21T12:46:12.758 INFO:teuthology.orchestra.run.smithi022.stdout:mon.smithi022 smithi022 running (6m) 115s ago 19m 50.6M 2048M 19.0.0-2267-g960fbbe4 19e12293512f c620c9071b2c 2024-03-21T12:46:12.758 INFO:teuthology.orchestra.run.smithi022.stdout:mon.smithi189 smithi189 running (5m) 40s ago 16m 43.3M 2048M 19.0.0-2267-g960fbbe4 19e12293512f b353af3d8f3b 2024-03-21T12:46:12.758 INFO:teuthology.orchestra.run.smithi022.stdout:nfs.foo.0.0.smithi022.ynhbbq smithi022 *:2049 running (11m) 115s ago 11m 277M - 4.0 e1d6a67b021e 33e48e64c60b 2024-03-21T12:46:12.758 INFO:teuthology.orchestra.run.smithi022.stdout:nfs.foo.1.0.smithi189.yzsqml smithi189 *:2049 running (11m) 40s ago 11m 86.0M - 4.0 e1d6a67b021e 48fab2cbdde2 2024-03-21T12:46:12.758 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.smithi022 smithi022 *:9100 running (8m) 115s ago 17m 18.2M - 1.3.1 1dbe0e931976 33287f803f4f 2024-03-21T12:46:12.759 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.smithi189 smithi189 *:9100 running (7m) 40s ago 16m 17.6M - 1.3.1 1dbe0e931976 b51d872e438a 2024-03-21T12:46:12.759 INFO:teuthology.orchestra.run.smithi022.stdout:osd.0 smithi022 running (4m) 115s ago 16m 336M 1937M 19.0.0-2267-g960fbbe4 19e12293512f 23a85645eeea 2024-03-21T12:46:12.759 INFO:teuthology.orchestra.run.smithi022.stdout:osd.1 smithi022 running (3m) 115s ago 15m 286M 1937M 19.0.0-2267-g960fbbe4 19e12293512f 0f90e70bbf0a 2024-03-21T12:46:12.759 INFO:teuthology.orchestra.run.smithi022.stdout:osd.2 smithi022 running (2m) 115s ago 15m 214M 1937M 19.0.0-2267-g960fbbe4 19e12293512f 78a229f593f8 2024-03-21T12:46:12.759 INFO:teuthology.orchestra.run.smithi022.stdout:osd.3 smithi022 running (2m) 115s ago 14m 12.5M 1937M 19.0.0-2267-g960fbbe4 19e12293512f 61a63b67d9af 2024-03-21T12:46:12.759 INFO:teuthology.orchestra.run.smithi022.stdout:osd.4 smithi189 running (82s) 40s ago 14m 270M 2704M 19.0.0-2267-g960fbbe4 19e12293512f 382755cee998 2024-03-21T12:46:12.759 INFO:teuthology.orchestra.run.smithi022.stdout:osd.5 smithi189 running (45s) 40s ago 13m 12.1M 2704M 19.0.0-2267-g960fbbe4 19e12293512f d2c15ba54b4f 2024-03-21T12:46:12.759 INFO:teuthology.orchestra.run.smithi022.stdout:osd.6 smithi189 running (13m) 40s ago 13m 554M 2704M 17.2.0 e1d6a67b021e 0c192fcfd65b 2024-03-21T12:46:12.759 INFO:teuthology.orchestra.run.smithi022.stdout:osd.7 smithi189 running (13m) 40s ago 13m 606M 2704M 17.2.0 e1d6a67b021e 3d0723e1f863 2024-03-21T12:46:12.759 INFO:teuthology.orchestra.run.smithi022.stdout:prometheus.smithi022 smithi022 *:9095 running (6m) 115s ago 16m 73.9M - 2.33.4 514e6a882f6e 8171bbe41d2c 2024-03-21T12:46:12.889 INFO:teuthology.orchestra.run.smithi022.stdout: 5 276604 9.91 MB/sec execute 552 sec latency 60.665 ms 2024-03-21T12:46:13.295 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-03-21T12:46:13.295 INFO:teuthology.orchestra.run.smithi022.stdout: "mon": { 2024-03-21T12:46:13.295 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 2 2024-03-21T12:46:13.295 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:46:13.295 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr": { 2024-03-21T12:46:13.295 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 2 2024-03-21T12:46:13.295 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:46:13.295 INFO:teuthology.orchestra.run.smithi022.stdout: "osd": { 2024-03-21T12:46:13.295 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-03-21T12:46:13.295 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 6 2024-03-21T12:46:13.296 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:46:13.296 INFO:teuthology.orchestra.run.smithi022.stdout: "mds": { 2024-03-21T12:46:13.296 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-21T12:46:13.296 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:46:13.296 INFO:teuthology.orchestra.run.smithi022.stdout: "overall": { 2024-03-21T12:46:13.296 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-03-21T12:46:13.296 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 10 2024-03-21T12:46:13.296 INFO:teuthology.orchestra.run.smithi022.stdout: } 2024-03-21T12:46:13.296 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-03-21T12:46:13.456 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:13 smithi022 ceph-mon[67047]: from='client.34412 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:46:13.457 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:13 smithi022 ceph-mon[67047]: from='client.34416 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:46:13.457 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:13 smithi022 ceph-mon[67047]: from='client.? 172.21.15.22:0/1812370672' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:46:13.777 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:46:13 smithi189 systemd[1]: Stopping Ceph osd.6 for 3fc7f0f2-e77e-11ee-95cd-87774f69a715... 2024-03-21T12:46:13.777 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:13 smithi189 ceph-mon[53160]: from='client.34412 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:46:13.777 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:13 smithi189 ceph-mon[53160]: from='client.34416 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:46:13.777 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:13 smithi189 ceph-mon[53160]: from='client.? 172.21.15.22:0/1812370672' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:46:13.789 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-03-21T12:46:13.789 INFO:teuthology.orchestra.run.smithi022.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93", 2024-03-21T12:46:13.789 INFO:teuthology.orchestra.run.smithi022.stdout: "in_progress": true, 2024-03-21T12:46:13.789 INFO:teuthology.orchestra.run.smithi022.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-21T12:46:13.789 INFO:teuthology.orchestra.run.smithi022.stdout: "services_complete": [ 2024-03-21T12:46:13.789 INFO:teuthology.orchestra.run.smithi022.stdout: "mon", 2024-03-21T12:46:13.789 INFO:teuthology.orchestra.run.smithi022.stdout: "crash", 2024-03-21T12:46:13.790 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr" 2024-03-21T12:46:13.790 INFO:teuthology.orchestra.run.smithi022.stdout: ], 2024-03-21T12:46:13.790 INFO:teuthology.orchestra.run.smithi022.stdout: "progress": "12/23 daemons upgraded", 2024-03-21T12:46:13.790 INFO:teuthology.orchestra.run.smithi022.stdout: "message": "Currently upgrading osd daemons", 2024-03-21T12:46:13.790 INFO:teuthology.orchestra.run.smithi022.stdout: "is_paused": false 2024-03-21T12:46:13.790 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-03-21T12:46:13.889 INFO:teuthology.orchestra.run.smithi022.stdout: 5 276829 9.90 MB/sec execute 553 sec latency 63.880 ms 2024-03-21T12:46:14.035 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:46:13 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6[37273]: 2024-03-21T12:46:13.775+0000 7f70a6290700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-21T12:46:14.036 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:46:13 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6[37273]: 2024-03-21T12:46:13.775+0000 7f70a6290700 -1 osd.6 140 *** Got signal Terminated *** 2024-03-21T12:46:14.036 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:46:13 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6[37273]: 2024-03-21T12:46:13.775+0000 7f70a6290700 -1 osd.6 140 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-21T12:46:14.369 INFO:teuthology.orchestra.run.smithi022.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-03-21T12:46:14.370 INFO:teuthology.orchestra.run.smithi022.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-21T12:46:14.370 INFO:teuthology.orchestra.run.smithi022.stdout: daemon grafana.smithi022 on smithi022 is in error state 2024-03-21T12:46:14.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:14 smithi022 ceph-mon[67047]: from='client.34420 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:46:14.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:14 smithi022 ceph-mon[67047]: pgmap v266: 97 pgs: 97 active+clean; 341 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 5.6 MiB/s wr, 559 op/s 2024-03-21T12:46:14.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:14 smithi022 ceph-mon[67047]: osd.6 marked itself down and dead 2024-03-21T12:46:14.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:14 smithi022 ceph-mon[67047]: from='client.34428 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:46:14.595 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:14 smithi022 ceph-mon[67047]: from='client.? 172.21.15.22:0/2495367031' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-21T12:46:14.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:14 smithi189 ceph-mon[53160]: from='client.34420 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:46:14.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:14 smithi189 ceph-mon[53160]: pgmap v266: 97 pgs: 97 active+clean; 341 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 5.6 MiB/s wr, 559 op/s 2024-03-21T12:46:14.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:14 smithi189 ceph-mon[53160]: osd.6 marked itself down and dead 2024-03-21T12:46:14.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:14 smithi189 ceph-mon[53160]: from='client.34428 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:46:14.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:14 smithi189 ceph-mon[53160]: from='client.? 172.21.15.22:0/2495367031' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-21T12:46:14.889 INFO:teuthology.orchestra.run.smithi022.stdout: 5 277012 9.90 MB/sec execute 554 sec latency 517.105 ms 2024-03-21T12:46:15.317 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:46:15 smithi189 podman[62171]: 2024-03-21 12:46:15.050811035 +0000 UTC m=+1.410250335 container died 0c192fcfd65b30a4d6396fc846ff7f6da2f13ca2c328b8337344628eb8f4236a (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, version=8, io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, vcs-type=git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, RELEASE=HEAD, name=centos-stream, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux ) 2024-03-21T12:46:15.629 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:15 smithi022 ceph-mon[67047]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-21T12:46:15.630 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:15 smithi022 ceph-mon[67047]: osdmap e141: 8 total, 7 up, 8 in 2024-03-21T12:46:15.630 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:15 smithi022 ceph-mon[67047]: pgmap v268: 97 pgs: 11 peering, 6 stale+active+clean, 80 active+clean; 337 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.7 MiB/s wr, 479 op/s 2024-03-21T12:46:15.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:15 smithi189 ceph-mon[53160]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-21T12:46:15.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:15 smithi189 ceph-mon[53160]: osdmap e141: 8 total, 7 up, 8 in 2024-03-21T12:46:15.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:15 smithi189 ceph-mon[53160]: pgmap v268: 97 pgs: 11 peering, 6 stale+active+clean, 80 active+clean; 337 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.7 MiB/s wr, 479 op/s 2024-03-21T12:46:15.786 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:46:15 smithi189 podman[62171]: 2024-03-21 12:46:15.418256406 +0000 UTC m=+1.777695711 container cleanup 0c192fcfd65b30a4d6396fc846ff7f6da2f13ca2c328b8337344628eb8f4236a (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, RELEASE=HEAD, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, GIT_BRANCH=HEAD, name=centos-stream, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-03-21T12:46:15.786 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:46:15 smithi189 bash[62171]: ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6 2024-03-21T12:46:15.889 INFO:teuthology.orchestra.run.smithi022.stdout: 5 277162 9.88 MB/sec execute 555 sec latency 1087.790 ms 2024-03-21T12:46:16.206 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:46:15 smithi189 podman[62187]: 2024-03-21 12:46:15.860255945 +0000 UTC m=+0.806846566 container remove 0c192fcfd65b30a4d6396fc846ff7f6da2f13ca2c328b8337344628eb8f4236a (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, io.buildah.version=1.19.8, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, ceph=True, vendor=Red Hat, Inc., release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, version=8, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-03-21T12:46:16.497 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:16 smithi189 ceph-mon[53160]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-03-21T12:46:16.497 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:16 smithi189 ceph-mon[53160]: osdmap e142: 8 total, 7 up, 8 in 2024-03-21T12:46:16.497 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:46:16 smithi189 podman[62279]: 2024-03-21 12:46:16.110547525 +0000 UTC m=+0.024559415 image pull 19e12293512ff7978c8adb5f72a829060dd47f07bfd6903b6b3178d2ebc29e4e quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93 2024-03-21T12:46:16.498 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:46:16 smithi189 podman[62279]: 2024-03-21 12:46:16.261546815 +0000 UTC m=+0.175558699 container create 0c598f94c186161f6728cfc9bbbbcf5d572e0e4d8605837ea9ee25c4fd5a9bcd (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, org.label-schema.build-date=20240319, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-03-21T12:46:16.636 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:16 smithi022 ceph-mon[67047]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-03-21T12:46:16.636 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:16 smithi022 ceph-mon[67047]: osdmap e142: 8 total, 7 up, 8 in 2024-03-21T12:46:16.786 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:46:16 smithi189 podman[62279]: 2024-03-21 12:46:16.497617127 +0000 UTC m=+0.411629002 container init 0c598f94c186161f6728cfc9bbbbcf5d572e0e4d8605837ea9ee25c4fd5a9bcd (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6-deactivate, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240319, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-03-21T12:46:16.786 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:46:16 smithi189 podman[62279]: 2024-03-21 12:46:16.501028266 +0000 UTC m=+0.415040144 container start 0c598f94c186161f6728cfc9bbbbcf5d572e0e4d8605837ea9ee25c4fd5a9bcd (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6-deactivate, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240319, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.33.5, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30) 2024-03-21T12:46:16.786 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:46:16 smithi189 podman[62279]: 2024-03-21 12:46:16.606046679 +0000 UTC m=+0.520058567 container attach 0c598f94c186161f6728cfc9bbbbcf5d572e0e4d8605837ea9ee25c4fd5a9bcd (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6-deactivate, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, org.label-schema.build-date=20240319, org.label-schema.license=GPLv2) 2024-03-21T12:46:16.889 INFO:teuthology.orchestra.run.smithi022.stdout: 5 277540 9.88 MB/sec execute 556 sec latency 46.720 ms 2024-03-21T12:46:17.264 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:46:16 smithi189 podman[62279]: 2024-03-21 12:46:16.849128775 +0000 UTC m=+0.763140651 container died 0c598f94c186161f6728cfc9bbbbcf5d572e0e4d8605837ea9ee25c4fd5a9bcd (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6-deactivate, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.build-date=20240319, maintainer=Guillaume Abrioux ) 2024-03-21T12:46:17.400 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:17 smithi022 ceph-mon[67047]: pgmap v270: 97 pgs: 6 active+undersized+degraded, 11 peering, 4 stale+active+clean, 76 active+clean; 339 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.0 KiB/s rd, 4.2 MiB/s wr, 393 op/s; 52/1671 objects degraded (3.112%) 2024-03-21T12:46:17.526 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:17 smithi189 ceph-mon[53160]: pgmap v270: 97 pgs: 6 active+undersized+degraded, 11 peering, 4 stale+active+clean, 76 active+clean; 339 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.0 KiB/s rd, 4.2 MiB/s wr, 393 op/s; 52/1671 objects degraded (3.112%) 2024-03-21T12:46:17.526 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:46:17 smithi189 podman[62305]: 2024-03-21 12:46:17.510362041 +0000 UTC m=+0.656322666 container remove 0c598f94c186161f6728cfc9bbbbcf5d572e0e4d8605837ea9ee25c4fd5a9bcd (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6-deactivate, GIT_CLEAN=True, org.label-schema.build-date=20240319, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-03-21T12:46:17.786 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:46:17 smithi189 systemd[1]: ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715@osd.6.service: Deactivated successfully. 2024-03-21T12:46:17.786 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:46:17 smithi189 systemd[1]: Stopped Ceph osd.6 for 3fc7f0f2-e77e-11ee-95cd-87774f69a715. 2024-03-21T12:46:17.786 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:46:17 smithi189 systemd[1]: ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715@osd.6.service: Consumed 5min 17.132s CPU time. 2024-03-21T12:46:17.786 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:46:17 smithi189 systemd[1]: Starting Ceph osd.6 for 3fc7f0f2-e77e-11ee-95cd-87774f69a715... 2024-03-21T12:46:17.889 INFO:teuthology.orchestra.run.smithi022.stdout: 5 277866 9.88 MB/sec execute 557 sec latency 41.828 ms 2024-03-21T12:46:18.286 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:46:18 smithi189 podman[62404]: 2024-03-21 12:46:17.908406792 +0000 UTC m=+0.021269474 image pull 19e12293512ff7978c8adb5f72a829060dd47f07bfd6903b6b3178d2ebc29e4e quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93 2024-03-21T12:46:18.286 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:46:18 smithi189 podman[62404]: 2024-03-21 12:46:18.037113711 +0000 UTC m=+0.149976398 container create 9b660b962bdab1cda75eec92e7bca8c777bf14d8a3cc01f2251d844bf4e4a367 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6-activate, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_CLEAN=True, ceph=True, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240319, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-21T12:46:18.415 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:18 smithi022 ceph-mon[67047]: Health check failed: Degraded data redundancy: 52/1671 objects degraded (3.112%), 6 pgs degraded (PG_DEGRADED) 2024-03-21T12:46:18.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:18 smithi189 ceph-mon[53160]: Health check failed: Degraded data redundancy: 52/1671 objects degraded (3.112%), 6 pgs degraded (PG_DEGRADED) 2024-03-21T12:46:18.786 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:46:18 smithi189 podman[62404]: 2024-03-21 12:46:18.315328977 +0000 UTC m=+0.428191659 container init 9b660b962bdab1cda75eec92e7bca8c777bf14d8a3cc01f2251d844bf4e4a367 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6-activate, ceph=True, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.build-date=20240319, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-03-21T12:46:18.786 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:46:18 smithi189 podman[62404]: 2024-03-21 12:46:18.318829864 +0000 UTC m=+0.431692555 container start 9b660b962bdab1cda75eec92e7bca8c777bf14d8a3cc01f2251d844bf4e4a367 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6-activate, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240319, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, ceph=True) 2024-03-21T12:46:18.786 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:46:18 smithi189 podman[62404]: 2024-03-21 12:46:18.414999847 +0000 UTC m=+0.527862544 container attach 9b660b962bdab1cda75eec92e7bca8c777bf14d8a3cc01f2251d844bf4e4a367 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240319, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0) 2024-03-21T12:46:18.890 INFO:teuthology.orchestra.run.smithi022.stdout: 5 278247 9.88 MB/sec execute 558 sec latency 47.626 ms 2024-03-21T12:46:19.219 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:46:18 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6-activate[62419]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-03-21T12:46:19.219 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:46:18 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6-activate[62419]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-03-21T12:46:19.219 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:46:18 smithi189 bash[62404]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-03-21T12:46:19.219 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:46:18 smithi189 bash[62404]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-03-21T12:46:19.219 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:46:18 smithi189 bash[62404]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-03-21T12:46:19.219 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:46:18 smithi189 bash[62404]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-03-21T12:46:19.219 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:46:18 smithi189 bash[62404]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-03-21T12:46:19.219 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:46:18 smithi189 bash[62404]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-03-21T12:46:19.219 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:46:18 smithi189 bash[62404]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-03-21T12:46:19.220 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:46:18 smithi189 bash[62404]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-03-21T12:46:19.220 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:46:18 smithi189 bash[62404]: --> ceph-volume raw activate successful for osd ID: 6 2024-03-21T12:46:19.220 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:46:18 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6-activate[62419]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-03-21T12:46:19.220 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:46:18 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6-activate[62419]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-03-21T12:46:19.220 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:46:18 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6-activate[62419]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-03-21T12:46:19.220 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:46:18 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6-activate[62419]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-03-21T12:46:19.220 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:46:18 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6-activate[62419]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-03-21T12:46:19.220 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:46:18 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6-activate[62419]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-03-21T12:46:19.220 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:46:18 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6-activate[62419]: --> ceph-volume raw activate successful for osd ID: 6 2024-03-21T12:46:19.220 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:46:18 smithi189 podman[62404]: 2024-03-21 12:46:18.901925689 +0000 UTC m=+1.014788379 container died 9b660b962bdab1cda75eec92e7bca8c777bf14d8a3cc01f2251d844bf4e4a367 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6-activate, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240319, CEPH_POINT_RELEASE=, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-03-21T12:46:19.449 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:19 smithi022 ceph-mon[67047]: pgmap v271: 97 pgs: 3 active+undersized, 14 active+undersized+degraded, 11 peering, 2 stale+active+clean, 67 active+clean; 336 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.2 KiB/s rd, 4.6 MiB/s wr, 448 op/s; 115/1635 objects degraded (7.034%); 501 B/s, 0 objects/s recovering 2024-03-21T12:46:19.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:19 smithi189 ceph-mon[53160]: pgmap v271: 97 pgs: 3 active+undersized, 14 active+undersized+degraded, 11 peering, 2 stale+active+clean, 67 active+clean; 336 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.2 KiB/s rd, 4.6 MiB/s wr, 448 op/s; 115/1635 objects degraded (7.034%); 501 B/s, 0 objects/s recovering 2024-03-21T12:46:19.803 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:46:19 smithi189 podman[62557]: 2024-03-21 12:46:19.536225203 +0000 UTC m=+0.624980001 container remove 9b660b962bdab1cda75eec92e7bca8c777bf14d8a3cc01f2251d844bf4e4a367 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6-activate, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, ceph=True, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240319, org.label-schema.license=GPLv2) 2024-03-21T12:46:19.803 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:46:19 smithi189 podman[62647]: 2024-03-21 12:46:19.679495736 +0000 UTC m=+0.034600716 image pull 19e12293512ff7978c8adb5f72a829060dd47f07bfd6903b6b3178d2ebc29e4e quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93 2024-03-21T12:46:19.804 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:46:19 smithi189 podman[62647]: 2024-03-21 12:46:19.803922907 +0000 UTC m=+0.159027874 container create 291089fb3596c5b9ee54f27a4b98f147108fa4aebca287ba916c16e5b718a9c8 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6, org.label-schema.build-date=20240319, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-21T12:46:19.890 INFO:teuthology.orchestra.run.smithi022.stdout: 5 278614 9.87 MB/sec execute 559 sec latency 32.299 ms 2024-03-21T12:46:20.066 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:46:20 smithi189 podman[62647]: 2024-03-21 12:46:20.039904998 +0000 UTC m=+0.395009976 container init 291089fb3596c5b9ee54f27a4b98f147108fa4aebca287ba916c16e5b718a9c8 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6, org.label-schema.vendor=CentOS, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240319, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_BRANCH=HEAD) 2024-03-21T12:46:20.066 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:46:20 smithi189 podman[62647]: 2024-03-21 12:46:20.043612694 +0000 UTC m=+0.398717663 container start 291089fb3596c5b9ee54f27a4b98f147108fa4aebca287ba916c16e5b718a9c8 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, ceph=True, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.build-date=20240319, CEPH_POINT_RELEASE=) 2024-03-21T12:46:20.536 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:46:20 smithi189 ceph-osd[62665]: -- 172.21.15.189:0/1504098894 <== mon.1 v2:172.21.15.189:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x560f2ae87680 con 0x560f2ae50c00 2024-03-21T12:46:20.536 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:46:20 smithi189 bash[62647]: 291089fb3596c5b9ee54f27a4b98f147108fa4aebca287ba916c16e5b718a9c8 2024-03-21T12:46:20.536 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:46:20 smithi189 systemd[1]: Started Ceph osd.6 for 3fc7f0f2-e77e-11ee-95cd-87774f69a715. 2024-03-21T12:46:20.890 INFO:teuthology.orchestra.run.smithi022.stdout: 5 278929 9.88 MB/sec execute 560 sec latency 45.612 ms 2024-03-21T12:46:21.036 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:46:20 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6[62661]: 2024-03-21T12:46:20.591+0000 7fbdbe0177c0 -1 Falling back to public interface 2024-03-21T12:46:21.276 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:21 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:46:21.277 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:21 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:46:21.277 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:21 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:46:21.277 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:21 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:46:21.277 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:21 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:46:21.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:21 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:46:21.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:21 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:46:21.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:21 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:46:21.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:21 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:46:21.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:21 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:46:21.890 INFO:teuthology.orchestra.run.smithi022.stdout: 5 279267 9.88 MB/sec execute 561 sec latency 53.291 ms 2024-03-21T12:46:22.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:22 smithi189 ceph-mon[53160]: pgmap v272: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 331 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.1 KiB/s rd, 2.7 MiB/s wr, 224 op/s; 194/1635 objects degraded (11.865%); 501 B/s, 0 objects/s recovering 2024-03-21T12:46:22.296 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:22 smithi022 ceph-mon[67047]: pgmap v272: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 331 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.1 KiB/s rd, 2.7 MiB/s wr, 224 op/s; 194/1635 objects degraded (11.865%); 501 B/s, 0 objects/s recovering 2024-03-21T12:46:22.890 INFO:teuthology.orchestra.run.smithi022.stdout: 5 279641 9.88 MB/sec execute 562 sec latency 51.348 ms 2024-03-21T12:46:23.311 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:23 smithi022 ceph-mon[67047]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-03-21T12:46:23.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:23 smithi189 ceph-mon[53160]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-03-21T12:46:23.890 INFO:teuthology.orchestra.run.smithi022.stdout: 5 280033 9.88 MB/sec execute 563 sec latency 51.299 ms 2024-03-21T12:46:24.316 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:24 smithi022 ceph-mon[67047]: pgmap v273: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 335 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 3.7 KiB/s rd, 4.7 MiB/s wr, 451 op/s; 196/1635 objects degraded (11.988%); 450 B/s, 0 objects/s recovering 2024-03-21T12:46:24.317 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:24 smithi022 ceph-mon[67047]: Health check update: Degraded data redundancy: 194/1635 objects degraded (11.865%), 25 pgs degraded (PG_DEGRADED) 2024-03-21T12:46:24.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:24 smithi189 ceph-mon[53160]: pgmap v273: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 335 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 3.7 KiB/s rd, 4.7 MiB/s wr, 451 op/s; 196/1635 objects degraded (11.988%); 450 B/s, 0 objects/s recovering 2024-03-21T12:46:24.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:24 smithi189 ceph-mon[53160]: Health check update: Degraded data redundancy: 194/1635 objects degraded (11.865%), 25 pgs degraded (PG_DEGRADED) 2024-03-21T12:46:24.890 INFO:teuthology.orchestra.run.smithi022.stdout: 5 280355 9.88 MB/sec execute 564 sec latency 45.286 ms 2024-03-21T12:46:25.335 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:25 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:46:25.336 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:25 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:46:25.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:25 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:46:25.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:25 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:46:25.890 INFO:teuthology.orchestra.run.smithi022.stdout: 5 280688 9.88 MB/sec execute 565 sec latency 51.131 ms 2024-03-21T12:46:26.378 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:26 smithi022 ceph-mon[67047]: pgmap v274: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 339 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.9 MiB/s wr, 519 op/s; 213/1689 objects degraded (12.611%); 400 B/s, 0 objects/s recovering 2024-03-21T12:46:26.379 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:26 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:46:26.379 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:26 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:46:26.660 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:26 smithi189 ceph-mon[53160]: pgmap v274: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 339 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.9 MiB/s wr, 519 op/s; 213/1689 objects degraded (12.611%); 400 B/s, 0 objects/s recovering 2024-03-21T12:46:26.660 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:26 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:46:26.660 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:26 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:46:26.890 INFO:teuthology.orchestra.run.smithi022.stdout: 5 281034 9.87 MB/sec execute 566 sec latency 57.052 ms 2024-03-21T12:46:27.890 INFO:teuthology.orchestra.run.smithi022.stdout: 5 281322 9.87 MB/sec execute 567 sec latency 57.719 ms 2024-03-21T12:46:28.468 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:28 smithi022 ceph-mon[67047]: pgmap v275: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.8 MiB/s wr, 472 op/s; 203/1674 objects degraded (12.127%); 337 B/s, 0 objects/s recovering 2024-03-21T12:46:28.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:28 smithi189 ceph-mon[53160]: pgmap v275: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.8 MiB/s wr, 472 op/s; 203/1674 objects degraded (12.127%); 337 B/s, 0 objects/s recovering 2024-03-21T12:46:28.890 INFO:teuthology.orchestra.run.smithi022.stdout: 5 281607 9.87 MB/sec execute 568 sec latency 58.212 ms 2024-03-21T12:46:29.478 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:29 smithi022 ceph-mon[67047]: Health check update: Degraded data redundancy: 203/1674 objects degraded (12.127%), 25 pgs degraded (PG_DEGRADED) 2024-03-21T12:46:29.740 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:46:29 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6[62661]: 2024-03-21T12:46:29.470+0000 7fbdbe0177c0 -1 osd.6 0 read_superblock omap replica is missing. 2024-03-21T12:46:29.740 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:29 smithi189 ceph-mon[53160]: Health check update: Degraded data redundancy: 203/1674 objects degraded (12.127%), 25 pgs degraded (PG_DEGRADED) 2024-03-21T12:46:29.890 INFO:teuthology.orchestra.run.smithi022.stdout: 5 281914 9.86 MB/sec execute 569 sec latency 67.761 ms 2024-03-21T12:46:30.517 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:30 smithi022 ceph-mon[67047]: pgmap v276: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 342 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.9 MiB/s wr, 514 op/s; 202/1716 objects degraded (11.772%); 333 B/s, 0 objects/s recovering 2024-03-21T12:46:30.518 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:30 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:46:30.518 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:30 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:46:30.518 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:30 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:46:30.518 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:30 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:46:30.518 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:30 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:46:30.518 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:30 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:46:30.518 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:30 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:46:30.518 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:30 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:46:30.518 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:30 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:46:30.518 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:30 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:46:30.518 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:30 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-03-21T12:46:30.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:30 smithi189 ceph-mon[53160]: pgmap v276: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 342 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.9 MiB/s wr, 514 op/s; 202/1716 objects degraded (11.772%); 333 B/s, 0 objects/s recovering 2024-03-21T12:46:30.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:30 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:46:30.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:30 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:46:30.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:30 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:46:30.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:30 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:46:30.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:30 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:46:30.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:30 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:46:30.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:30 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:46:30.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:30 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:46:30.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:30 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:46:30.787 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:30 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:46:30.787 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:30 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-03-21T12:46:30.890 INFO:teuthology.orchestra.run.smithi022.stdout: 5 282229 9.85 MB/sec execute 570 sec latency 40.044 ms 2024-03-21T12:46:31.285 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:46:30 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6[62661]: 2024-03-21T12:46:30.848+0000 7fbdbe0177c0 -1 osd.6 140 log_to_monitors true 2024-03-21T12:46:31.533 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:31 smithi022 ceph-mon[67047]: pgmap v277: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 338 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 5.4 MiB/s wr, 559 op/s; 193/1653 objects degraded (11.676%) 2024-03-21T12:46:31.534 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:31 smithi022 ceph-mon[67047]: pgmap v278: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 338 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 6.3 MiB/s wr, 663 op/s; 193/1653 objects degraded (11.676%) 2024-03-21T12:46:31.534 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:31 smithi022 ceph-mon[67047]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-03-21T12:46:31.534 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:31 smithi022 ceph-mon[67047]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-03-21T12:46:31.534 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:31 smithi022 ceph-mon[67047]: from='osd.6 [v2:172.21.15.189:6816/1794412900,v1:172.21.15.189:6817/1794412900]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-03-21T12:46:31.534 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:31 smithi022 ceph-mon[67047]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-03-21T12:46:31.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:31 smithi189 ceph-mon[53160]: pgmap v277: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 338 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 5.4 MiB/s wr, 559 op/s; 193/1653 objects degraded (11.676%) 2024-03-21T12:46:31.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:31 smithi189 ceph-mon[53160]: pgmap v278: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 338 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 6.3 MiB/s wr, 663 op/s; 193/1653 objects degraded (11.676%) 2024-03-21T12:46:31.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:31 smithi189 ceph-mon[53160]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-03-21T12:46:31.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:31 smithi189 ceph-mon[53160]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-03-21T12:46:31.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:31 smithi189 ceph-mon[53160]: from='osd.6 [v2:172.21.15.189:6816/1794412900,v1:172.21.15.189:6817/1794412900]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-03-21T12:46:31.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:31 smithi189 ceph-mon[53160]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-03-21T12:46:31.890 INFO:teuthology.orchestra.run.smithi022.stdout: 5 282495 9.85 MB/sec execute 571 sec latency 47.735 ms 2024-03-21T12:46:32.285 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:46:31 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6[62661]: 2024-03-21T12:46:31.836+0000 7fbdaf46f700 -1 osd.6 140 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-21T12:46:32.890 INFO:teuthology.orchestra.run.smithi022.stdout: 5 282792 9.86 MB/sec execute 572 sec latency 84.856 ms 2024-03-21T12:46:33.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:32 smithi022 ceph-mon[67047]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-03-21T12:46:33.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:32 smithi022 ceph-mon[67047]: osdmap e143: 8 total, 7 up, 8 in 2024-03-21T12:46:33.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:32 smithi022 ceph-mon[67047]: from='osd.6 [v2:172.21.15.189:6816/1794412900,v1:172.21.15.189:6817/1794412900]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-03-21T12:46:33.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:32 smithi022 ceph-mon[67047]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-03-21T12:46:33.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:32 smithi022 ceph-mon[67047]: pgmap v280: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 335 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.1 MiB/s wr, 337 op/s; 194/1656 objects degraded (11.715%) 2024-03-21T12:46:33.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:32 smithi189 ceph-mon[53160]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-03-21T12:46:33.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:32 smithi189 ceph-mon[53160]: osdmap e143: 8 total, 7 up, 8 in 2024-03-21T12:46:33.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:32 smithi189 ceph-mon[53160]: from='osd.6 [v2:172.21.15.189:6816/1794412900,v1:172.21.15.189:6817/1794412900]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-03-21T12:46:33.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:32 smithi189 ceph-mon[53160]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-03-21T12:46:33.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:32 smithi189 ceph-mon[53160]: pgmap v280: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 335 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.1 MiB/s wr, 337 op/s; 194/1656 objects degraded (11.715%) 2024-03-21T12:46:33.890 INFO:teuthology.orchestra.run.smithi022.stdout: 5 282796 9.84 MB/sec execute 573 sec latency 1078.014 ms 2024-03-21T12:46:34.071 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:33 smithi022 ceph-mon[67047]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-21T12:46:34.071 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:33 smithi022 ceph-mon[67047]: osd.6 [v2:172.21.15.189:6816/1794412900,v1:172.21.15.189:6817/1794412900] boot 2024-03-21T12:46:34.071 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:33 smithi022 ceph-mon[67047]: osdmap e144: 8 total, 8 up, 8 in 2024-03-21T12:46:34.071 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:33 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-21T12:46:34.071 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:33 smithi022 ceph-mon[67047]: Health check update: Degraded data redundancy: 194/1656 objects degraded (11.715%), 25 pgs degraded (PG_DEGRADED) 2024-03-21T12:46:34.071 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:33 smithi022 ceph-mon[67047]: osdmap e145: 8 total, 8 up, 8 in 2024-03-21T12:46:34.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:33 smithi189 ceph-mon[53160]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-21T12:46:34.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:33 smithi189 ceph-mon[53160]: osd.6 [v2:172.21.15.189:6816/1794412900,v1:172.21.15.189:6817/1794412900] boot 2024-03-21T12:46:34.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:33 smithi189 ceph-mon[53160]: osdmap e144: 8 total, 8 up, 8 in 2024-03-21T12:46:34.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:33 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-21T12:46:34.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:33 smithi189 ceph-mon[53160]: Health check update: Degraded data redundancy: 194/1656 objects degraded (11.715%), 25 pgs degraded (PG_DEGRADED) 2024-03-21T12:46:34.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:33 smithi189 ceph-mon[53160]: osdmap e145: 8 total, 8 up, 8 in 2024-03-21T12:46:34.827 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:34 smithi022 ceph-mon[67047]: pgmap v283: 97 pgs: 9 peering, 7 active+undersized, 19 active+undersized+degraded, 62 active+clean; 336 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s wr, 399 op/s; 139/1578 objects degraded (8.809%) 2024-03-21T12:46:34.828 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:34 smithi022 ceph-mon[67047]: osdmap e146: 8 total, 8 up, 8 in 2024-03-21T12:46:34.890 INFO:teuthology.orchestra.run.smithi022.stdout: 5 282936 9.83 MB/sec execute 574 sec latency 995.326 ms 2024-03-21T12:46:35.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:34 smithi189 ceph-mon[53160]: pgmap v283: 97 pgs: 9 peering, 7 active+undersized, 19 active+undersized+degraded, 62 active+clean; 336 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s wr, 399 op/s; 139/1578 objects degraded (8.809%) 2024-03-21T12:46:35.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:34 smithi189 ceph-mon[53160]: osdmap e146: 8 total, 8 up, 8 in 2024-03-21T12:46:35.890 INFO:teuthology.orchestra.run.smithi022.stdout: 5 283263 9.82 MB/sec execute 575 sec latency 67.065 ms 2024-03-21T12:46:36.890 INFO:teuthology.orchestra.run.smithi022.stdout: 5 283458 9.82 MB/sec execute 576 sec latency 860.076 ms 2024-03-21T12:46:36.906 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:36 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:46:36.906 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:36 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:46:36.906 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:36 smithi022 ceph-mon[67047]: osdmap e147: 8 total, 8 up, 8 in 2024-03-21T12:46:36.907 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:36 smithi022 ceph-mon[67047]: pgmap v285: 97 pgs: 2 remapped+peering, 9 peering, 1 active+undersized, 13 active+undersized+degraded, 72 active+clean; 336 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.5 MiB/s wr, 365 op/s; 79/1572 objects degraded (5.025%); 15 KiB/s, 8 objects/s recovering 2024-03-21T12:46:37.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:36 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:46:37.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:36 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:46:37.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:36 smithi189 ceph-mon[53160]: osdmap e147: 8 total, 8 up, 8 in 2024-03-21T12:46:37.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:36 smithi189 ceph-mon[53160]: pgmap v285: 97 pgs: 2 remapped+peering, 9 peering, 1 active+undersized, 13 active+undersized+degraded, 72 active+clean; 336 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.5 MiB/s wr, 365 op/s; 79/1572 objects degraded (5.025%); 15 KiB/s, 8 objects/s recovering 2024-03-21T12:46:37.890 INFO:teuthology.orchestra.run.smithi022.stdout: 5 283700 9.81 MB/sec execute 577 sec latency 905.754 ms 2024-03-21T12:46:37.916 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:37 smithi022 ceph-mon[67047]: osdmap e148: 8 total, 8 up, 8 in 2024-03-21T12:46:38.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:37 smithi189 ceph-mon[53160]: osdmap e148: 8 total, 8 up, 8 in 2024-03-21T12:46:38.890 INFO:teuthology.orchestra.run.smithi022.stdout: 5 283966 9.80 MB/sec execute 578 sec latency 62.166 ms 2024-03-21T12:46:38.932 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:38 smithi022 ceph-mon[67047]: osdmap e149: 8 total, 8 up, 8 in 2024-03-21T12:46:38.933 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:38 smithi022 ceph-mon[67047]: pgmap v289: 97 pgs: 2 remapped+peering, 4 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 331 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 9.8 KiB/s rd, 3.3 MiB/s wr, 328 op/s; 29/1563 objects degraded (1.855%); 1.3 MiB/s, 35 objects/s recovering 2024-03-21T12:46:38.933 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:38 smithi022 ceph-mon[67047]: Health check update: Degraded data redundancy: 79/1572 objects degraded (5.025%), 13 pgs degraded (PG_DEGRADED) 2024-03-21T12:46:38.933 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:38 smithi022 ceph-mon[67047]: osdmap e150: 8 total, 8 up, 8 in 2024-03-21T12:46:39.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:38 smithi189 ceph-mon[53160]: osdmap e149: 8 total, 8 up, 8 in 2024-03-21T12:46:39.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:38 smithi189 ceph-mon[53160]: pgmap v289: 97 pgs: 2 remapped+peering, 4 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 331 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 9.8 KiB/s rd, 3.3 MiB/s wr, 328 op/s; 29/1563 objects degraded (1.855%); 1.3 MiB/s, 35 objects/s recovering 2024-03-21T12:46:39.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:38 smithi189 ceph-mon[53160]: Health check update: Degraded data redundancy: 79/1572 objects degraded (5.025%), 13 pgs degraded (PG_DEGRADED) 2024-03-21T12:46:39.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:38 smithi189 ceph-mon[53160]: osdmap e150: 8 total, 8 up, 8 in 2024-03-21T12:46:39.890 INFO:teuthology.orchestra.run.smithi022.stdout: 5 284177 9.79 MB/sec execute 579 sec latency 53.608 ms 2024-03-21T12:46:40.891 INFO:teuthology.orchestra.run.smithi022.stdout: 5 284454 9.79 MB/sec execute 580 sec latency 49.335 ms 2024-03-21T12:46:40.994 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:40 smithi022 ceph-mon[67047]: osdmap e151: 8 total, 8 up, 8 in 2024-03-21T12:46:40.994 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:40 smithi022 ceph-mon[67047]: pgmap v292: 97 pgs: 1 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 330 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 3.3 MiB/s wr, 361 op/s; 29/1575 objects degraded (1.841%); 1.7 MiB/s, 61 keys/s, 43 objects/s recovering 2024-03-21T12:46:41.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:40 smithi189 ceph-mon[53160]: osdmap e151: 8 total, 8 up, 8 in 2024-03-21T12:46:41.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:40 smithi189 ceph-mon[53160]: pgmap v292: 97 pgs: 1 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 330 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 3.3 MiB/s wr, 361 op/s; 29/1575 objects degraded (1.841%); 1.7 MiB/s, 61 keys/s, 43 objects/s recovering 2024-03-21T12:46:41.891 INFO:teuthology.orchestra.run.smithi022.stdout: 5 284780 9.79 MB/sec execute 581 sec latency 60.633 ms 2024-03-21T12:46:42.292 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:42 smithi022 ceph-mon[67047]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 29/1575 objects degraded (1.841%), 1 pg degraded) 2024-03-21T12:46:42.469 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:42 smithi189 ceph-mon[53160]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 29/1575 objects degraded (1.841%), 1 pg degraded) 2024-03-21T12:46:42.890 INFO:teuthology.orchestra.run.smithi022.stdout: 5 285105 9.79 MB/sec execute 582 sec latency 52.591 ms 2024-03-21T12:46:43.330 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:43 smithi022 ceph-mon[67047]: pgmap v293: 97 pgs: 97 active+clean; 333 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 3.9 MiB/s wr, 537 op/s; 527 KiB/s, 47 keys/s, 34 objects/s recovering 2024-03-21T12:46:43.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:43 smithi189 ceph-mon[53160]: pgmap v293: 97 pgs: 97 active+clean; 333 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 3.9 MiB/s wr, 537 op/s; 527 KiB/s, 47 keys/s, 34 objects/s recovering 2024-03-21T12:46:43.890 INFO:teuthology.orchestra.run.smithi022.stdout: 5 285416 9.80 MB/sec execute 583 sec latency 55.733 ms 2024-03-21T12:46:44.891 INFO:teuthology.orchestra.run.smithi022.stdout: 5 285673 9.79 MB/sec execute 584 sec latency 67.988 ms 2024-03-21T12:46:44.891 INFO:teuthology.orchestra.run.smithi022.stdout:true 2024-03-21T12:46:45.354 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:45 smithi022 ceph-mon[67047]: pgmap v294: 97 pgs: 97 active+clean; 334 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 4.4 KiB/s rd, 3.5 MiB/s wr, 350 op/s; 390 KiB/s, 39 keys/s, 20 objects/s recovering 2024-03-21T12:46:45.355 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:45 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-03-21T12:46:45.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:45 smithi189 ceph-mon[53160]: pgmap v294: 97 pgs: 97 active+clean; 334 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 4.4 KiB/s rd, 3.5 MiB/s wr, 350 op/s; 390 KiB/s, 39 keys/s, 20 objects/s recovering 2024-03-21T12:46:45.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:45 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-03-21T12:46:45.829 INFO:teuthology.orchestra.run.smithi022.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-21T12:46:45.829 INFO:teuthology.orchestra.run.smithi022.stdout:alertmanager.smithi022 smithi022 *:9093,9094 running (9m) 2m ago 18m 24.1M - 0.23.0 ba2b418f427c 14eb2a3ca318 2024-03-21T12:46:45.829 INFO:teuthology.orchestra.run.smithi022.stdout:crash.smithi022 smithi022 running (5m) 2m ago 18m 6660k - 19.0.0-2267-g960fbbe4 19e12293512f 356b2ddb609a 2024-03-21T12:46:45.829 INFO:teuthology.orchestra.run.smithi022.stdout:crash.smithi189 smithi189 running (5m) 21s ago 17m 6660k - 19.0.0-2267-g960fbbe4 19e12293512f 90ee837f0500 2024-03-21T12:46:45.830 INFO:teuthology.orchestra.run.smithi022.stdout:grafana.smithi022 smithi022 *:3000 error 2m ago 17m - - 2024-03-21T12:46:45.830 INFO:teuthology.orchestra.run.smithi022.stdout:mds.foofs.smithi022.bcnxho smithi022 running (12m) 2m ago 12m 38.3M - 17.2.0 e1d6a67b021e db26b67b8ee7 2024-03-21T12:46:45.830 INFO:teuthology.orchestra.run.smithi022.stdout:mds.foofs.smithi189.lluvko smithi189 running (12m) 21s ago 12m 17.8M - 17.2.0 e1d6a67b021e cf153aff1387 2024-03-21T12:46:45.830 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.smithi022.fektzs smithi022 *:8443,9283,8765 running (8m) 2m ago 19m 506M - 19.0.0-2267-g960fbbe4 19e12293512f d57cebd306b3 2024-03-21T12:46:45.830 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.smithi189.cnxzvu smithi189 *:8443,9283,8765 running (7m) 21s ago 17m 452M - 19.0.0-2267-g960fbbe4 19e12293512f 41663955c56f 2024-03-21T12:46:45.830 INFO:teuthology.orchestra.run.smithi022.stdout:mon.smithi022 smithi022 running (6m) 2m ago 19m 50.6M 2048M 19.0.0-2267-g960fbbe4 19e12293512f c620c9071b2c 2024-03-21T12:46:45.830 INFO:teuthology.orchestra.run.smithi022.stdout:mon.smithi189 smithi189 running (6m) 21s ago 17m 43.7M 2048M 19.0.0-2267-g960fbbe4 19e12293512f b353af3d8f3b 2024-03-21T12:46:45.830 INFO:teuthology.orchestra.run.smithi022.stdout:nfs.foo.0.0.smithi022.ynhbbq smithi022 *:2049 running (11m) 2m ago 11m 277M - 4.0 e1d6a67b021e 33e48e64c60b 2024-03-21T12:46:45.830 INFO:teuthology.orchestra.run.smithi022.stdout:nfs.foo.1.0.smithi189.yzsqml smithi189 *:2049 running (11m) 21s ago 11m 86.0M - 4.0 e1d6a67b021e 48fab2cbdde2 2024-03-21T12:46:45.830 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.smithi022 smithi022 *:9100 running (8m) 2m ago 17m 18.2M - 1.3.1 1dbe0e931976 33287f803f4f 2024-03-21T12:46:45.830 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.smithi189 smithi189 *:9100 running (8m) 21s ago 17m 17.3M - 1.3.1 1dbe0e931976 b51d872e438a 2024-03-21T12:46:45.830 INFO:teuthology.orchestra.run.smithi022.stdout:osd.0 smithi022 running (4m) 2m ago 16m 336M 1937M 19.0.0-2267-g960fbbe4 19e12293512f 23a85645eeea 2024-03-21T12:46:45.830 INFO:teuthology.orchestra.run.smithi022.stdout:osd.1 smithi022 running (4m) 2m ago 16m 286M 1937M 19.0.0-2267-g960fbbe4 19e12293512f 0f90e70bbf0a 2024-03-21T12:46:45.830 INFO:teuthology.orchestra.run.smithi022.stdout:osd.2 smithi022 running (3m) 2m ago 15m 214M 1937M 19.0.0-2267-g960fbbe4 19e12293512f 78a229f593f8 2024-03-21T12:46:45.830 INFO:teuthology.orchestra.run.smithi022.stdout:osd.3 smithi022 running (2m) 2m ago 15m 12.5M 1937M 19.0.0-2267-g960fbbe4 19e12293512f 61a63b67d9af 2024-03-21T12:46:45.831 INFO:teuthology.orchestra.run.smithi022.stdout:osd.4 smithi189 running (116s) 21s ago 14m 305M 2704M 19.0.0-2267-g960fbbe4 19e12293512f 382755cee998 2024-03-21T12:46:45.831 INFO:teuthology.orchestra.run.smithi022.stdout:osd.5 smithi189 running (78s) 21s ago 14m 274M 2704M 19.0.0-2267-g960fbbe4 19e12293512f d2c15ba54b4f 2024-03-21T12:46:45.831 INFO:teuthology.orchestra.run.smithi022.stdout:osd.6 smithi189 running (26s) 21s ago 14m 12.3M 2704M 19.0.0-2267-g960fbbe4 19e12293512f 291089fb3596 2024-03-21T12:46:45.831 INFO:teuthology.orchestra.run.smithi022.stdout:osd.7 smithi189 running (13m) 21s ago 13m 677M 2704M 17.2.0 e1d6a67b021e 3d0723e1f863 2024-03-21T12:46:45.831 INFO:teuthology.orchestra.run.smithi022.stdout:prometheus.smithi022 smithi022 *:9095 running (7m) 2m ago 16m 73.9M - 2.33.4 514e6a882f6e 8171bbe41d2c 2024-03-21T12:46:45.891 INFO:teuthology.orchestra.run.smithi022.stdout: 5 285952 9.78 MB/sec execute 585 sec latency 67.238 ms 2024-03-21T12:46:46.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:46 smithi189 ceph-mon[53160]: from='client.34440 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:46:46.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:46 smithi189 ceph-mon[53160]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-03-21T12:46:46.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:46 smithi189 ceph-mon[53160]: Upgrade: osd.7 is safe to restart 2024-03-21T12:46:46.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:46 smithi189 ceph-mon[53160]: from='client.34444 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:46:46.334 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:46 smithi022 ceph-mon[67047]: from='client.34440 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:46:46.334 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:46 smithi022 ceph-mon[67047]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-03-21T12:46:46.334 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:46 smithi022 ceph-mon[67047]: Upgrade: osd.7 is safe to restart 2024-03-21T12:46:46.334 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:46 smithi022 ceph-mon[67047]: from='client.34444 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:46:46.391 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-03-21T12:46:46.391 INFO:teuthology.orchestra.run.smithi022.stdout: "mon": { 2024-03-21T12:46:46.392 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 2 2024-03-21T12:46:46.392 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:46:46.392 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr": { 2024-03-21T12:46:46.392 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 2 2024-03-21T12:46:46.392 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:46:46.392 INFO:teuthology.orchestra.run.smithi022.stdout: "osd": { 2024-03-21T12:46:46.392 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-03-21T12:46:46.392 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 7 2024-03-21T12:46:46.392 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:46:46.392 INFO:teuthology.orchestra.run.smithi022.stdout: "mds": { 2024-03-21T12:46:46.392 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-21T12:46:46.392 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:46:46.392 INFO:teuthology.orchestra.run.smithi022.stdout: "overall": { 2024-03-21T12:46:46.392 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-03-21T12:46:46.393 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 11 2024-03-21T12:46:46.393 INFO:teuthology.orchestra.run.smithi022.stdout: } 2024-03-21T12:46:46.393 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-03-21T12:46:46.891 INFO:teuthology.orchestra.run.smithi022.stdout: 5 286233 9.78 MB/sec execute 586 sec latency 38.785 ms 2024-03-21T12:46:46.898 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-03-21T12:46:46.898 INFO:teuthology.orchestra.run.smithi022.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93", 2024-03-21T12:46:46.898 INFO:teuthology.orchestra.run.smithi022.stdout: "in_progress": true, 2024-03-21T12:46:46.898 INFO:teuthology.orchestra.run.smithi022.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-21T12:46:46.898 INFO:teuthology.orchestra.run.smithi022.stdout: "services_complete": [ 2024-03-21T12:46:46.898 INFO:teuthology.orchestra.run.smithi022.stdout: "mon", 2024-03-21T12:46:46.898 INFO:teuthology.orchestra.run.smithi022.stdout: "crash", 2024-03-21T12:46:46.898 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr" 2024-03-21T12:46:46.899 INFO:teuthology.orchestra.run.smithi022.stdout: ], 2024-03-21T12:46:46.899 INFO:teuthology.orchestra.run.smithi022.stdout: "progress": "13/23 daemons upgraded", 2024-03-21T12:46:46.899 INFO:teuthology.orchestra.run.smithi022.stdout: "message": "Currently upgrading osd daemons", 2024-03-21T12:46:46.899 INFO:teuthology.orchestra.run.smithi022.stdout: "is_paused": false 2024-03-21T12:46:46.899 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-03-21T12:46:47.394 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:47 smithi022 ceph-mon[67047]: from='client.34448 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:46:47.394 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:47 smithi022 ceph-mon[67047]: pgmap v295: 97 pgs: 97 active+clean; 335 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 3.1 KiB/s rd, 3.0 MiB/s wr, 282 op/s; 139 KiB/s, 30 keys/s, 11 objects/s recovering 2024-03-21T12:46:47.394 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:47 smithi022 ceph-mon[67047]: from='client.? 172.21.15.22:0/2664685511' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:46:47.394 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:47 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:46:47.394 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:47 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-03-21T12:46:47.394 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:47 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:46:47.443 INFO:teuthology.orchestra.run.smithi022.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-03-21T12:46:47.443 INFO:teuthology.orchestra.run.smithi022.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-21T12:46:47.443 INFO:teuthology.orchestra.run.smithi022.stdout: daemon grafana.smithi022 on smithi022 is in error state 2024-03-21T12:46:47.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:47 smithi189 ceph-mon[53160]: from='client.34448 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:46:47.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:47 smithi189 ceph-mon[53160]: pgmap v295: 97 pgs: 97 active+clean; 335 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 3.1 KiB/s rd, 3.0 MiB/s wr, 282 op/s; 139 KiB/s, 30 keys/s, 11 objects/s recovering 2024-03-21T12:46:47.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:47 smithi189 ceph-mon[53160]: from='client.? 172.21.15.22:0/2664685511' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:46:47.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:47 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:46:47.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:47 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-03-21T12:46:47.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:47 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:46:47.891 INFO:teuthology.orchestra.run.smithi022.stdout: 5 286485 9.77 MB/sec execute 587 sec latency 49.500 ms 2024-03-21T12:46:48.153 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:48 smithi022 ceph-mon[67047]: from='client.34456 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:46:48.154 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:48 smithi022 ceph-mon[67047]: Upgrade: Updating osd.7 2024-03-21T12:46:48.154 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:48 smithi022 ceph-mon[67047]: Deploying daemon osd.7 on smithi189 2024-03-21T12:46:48.154 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:48 smithi022 ceph-mon[67047]: from='client.? 172.21.15.22:0/2969460934' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-21T12:46:48.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:48 smithi189 ceph-mon[53160]: from='client.34456 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:46:48.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:48 smithi189 ceph-mon[53160]: Upgrade: Updating osd.7 2024-03-21T12:46:48.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:48 smithi189 ceph-mon[53160]: Deploying daemon osd.7 on smithi189 2024-03-21T12:46:48.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:48 smithi189 ceph-mon[53160]: from='client.? 172.21.15.22:0/2969460934' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-21T12:46:48.891 INFO:teuthology.orchestra.run.smithi022.stdout: 5 286800 9.77 MB/sec execute 588 sec latency 54.765 ms 2024-03-21T12:46:49.165 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:49 smithi022 ceph-mon[67047]: pgmap v296: 97 pgs: 97 active+clean; 342 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.4 MiB/s wr, 437 op/s; 104 KiB/s, 7 objects/s recovering 2024-03-21T12:46:49.185 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:49 smithi189 ceph-mon[53160]: pgmap v296: 97 pgs: 97 active+clean; 342 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.4 MiB/s wr, 437 op/s; 104 KiB/s, 7 objects/s recovering 2024-03-21T12:46:49.891 INFO:teuthology.orchestra.run.smithi022.stdout: 5 287181 9.77 MB/sec execute 589 sec latency 49.807 ms 2024-03-21T12:46:50.891 INFO:teuthology.orchestra.run.smithi022.stdout: 5 287495 9.76 MB/sec execute 590 sec latency 65.113 ms 2024-03-21T12:46:51.036 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:46:50 smithi189 systemd[1]: Stopping Ceph osd.7 for 3fc7f0f2-e77e-11ee-95cd-87774f69a715... 2024-03-21T12:46:51.220 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:51 smithi022 ceph-mon[67047]: pgmap v297: 97 pgs: 97 active+clean; 336 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.3 MiB/s wr, 415 op/s; 0 B/s, 2 objects/s recovering 2024-03-21T12:46:51.220 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:51 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:46:51.220 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:51 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:46:51.220 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:51 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:46:51.220 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:51 smithi022 ceph-mon[67047]: osd.7 marked itself down and dead 2024-03-21T12:46:51.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:51 smithi189 ceph-mon[53160]: pgmap v297: 97 pgs: 97 active+clean; 336 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.3 MiB/s wr, 415 op/s; 0 B/s, 2 objects/s recovering 2024-03-21T12:46:51.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:51 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:46:51.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:51 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:46:51.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:51 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:46:51.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:51 smithi189 ceph-mon[53160]: osd.7 marked itself down and dead 2024-03-21T12:46:51.536 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:46:51 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[40286]: 2024-03-21T12:46:51.060+0000 7f0ab60b2700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-21T12:46:51.536 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:46:51 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[40286]: 2024-03-21T12:46:51.060+0000 7f0ab60b2700 -1 osd.7 151 *** Got signal Terminated *** 2024-03-21T12:46:51.536 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:46:51 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[40286]: 2024-03-21T12:46:51.060+0000 7f0ab60b2700 -1 osd.7 151 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-21T12:46:51.891 INFO:teuthology.orchestra.run.smithi022.stdout: 5 287786 9.76 MB/sec execute 591 sec latency 41.492 ms 2024-03-21T12:46:52.240 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:52 smithi022 ceph-mon[67047]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-21T12:46:52.241 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:52 smithi022 ceph-mon[67047]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-03-21T12:46:52.241 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:52 smithi022 ceph-mon[67047]: osdmap e152: 8 total, 7 up, 8 in 2024-03-21T12:46:52.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:52 smithi189 ceph-mon[53160]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-21T12:46:52.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:52 smithi189 ceph-mon[53160]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-03-21T12:46:52.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:52 smithi189 ceph-mon[53160]: osdmap e152: 8 total, 7 up, 8 in 2024-03-21T12:46:52.891 INFO:teuthology.orchestra.run.smithi022.stdout: 5 287923 9.75 MB/sec execute 592 sec latency 801.867 ms 2024-03-21T12:46:53.000 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:46:52 smithi189 podman[64439]: 2024-03-21 12:46:52.728617257 +0000 UTC m=+1.803118374 container died 3d0723e1f863a22922ff460b86dd64c7674fe70e3555133f90f82c24416ba1ad (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, release=754, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, vcs-type=git, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD) 2024-03-21T12:46:53.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:53 smithi189 ceph-mon[53160]: pgmap v298: 97 pgs: 97 active+clean; 338 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.3 MiB/s wr, 441 op/s; 0 B/s, 2 objects/s recovering 2024-03-21T12:46:53.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:53 smithi189 ceph-mon[53160]: osdmap e153: 8 total, 7 up, 8 in 2024-03-21T12:46:53.286 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:46:53 smithi189 podman[64439]: 2024-03-21 12:46:53.082391324 +0000 UTC m=+2.156892445 container cleanup 3d0723e1f863a22922ff460b86dd64c7674fe70e3555133f90f82c24416ba1ad (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, io.buildah.version=1.19.8, GIT_BRANCH=HEAD, RELEASE=HEAD, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-03-21T12:46:53.286 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:46:53 smithi189 bash[64439]: ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7 2024-03-21T12:46:53.323 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:53 smithi022 ceph-mon[67047]: pgmap v298: 97 pgs: 97 active+clean; 338 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.3 MiB/s wr, 441 op/s; 0 B/s, 2 objects/s recovering 2024-03-21T12:46:53.323 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:53 smithi022 ceph-mon[67047]: osdmap e153: 8 total, 7 up, 8 in 2024-03-21T12:46:53.735 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:46:53 smithi189 podman[64458]: 2024-03-21 12:46:53.392223121 +0000 UTC m=+0.664523726 container remove 3d0723e1f863a22922ff460b86dd64c7674fe70e3555133f90f82c24416ba1ad (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7, maintainer=Guillaume Abrioux , io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., ceph=True, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD) 2024-03-21T12:46:53.891 INFO:teuthology.orchestra.run.smithi022.stdout: 5 288235 9.75 MB/sec execute 593 sec latency 980.128 ms 2024-03-21T12:46:54.029 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:46:53 smithi189 podman[64546]: 2024-03-21 12:46:53.638580353 +0000 UTC m=+0.018957798 image pull 19e12293512ff7978c8adb5f72a829060dd47f07bfd6903b6b3178d2ebc29e4e quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93 2024-03-21T12:46:54.029 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:46:53 smithi189 podman[64546]: 2024-03-21 12:46:53.76081451 +0000 UTC m=+0.141192004 container create 120187b22491d4e2edea617c696f6f032b35a54a624ceb7f5c916ee9ea55d7a7 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7-deactivate, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240319, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-21T12:46:54.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:54 smithi189 ceph-mon[53160]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-03-21T12:46:54.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:54 smithi189 ceph-mon[53160]: Health check failed: Degraded data redundancy: 16/1635 objects degraded (0.979%), 4 pgs degraded (PG_DEGRADED) 2024-03-21T12:46:54.286 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:46:54 smithi189 podman[64546]: 2024-03-21 12:46:54.029722564 +0000 UTC m=+0.410100020 container init 120187b22491d4e2edea617c696f6f032b35a54a624ceb7f5c916ee9ea55d7a7 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7-deactivate, CEPH_POINT_RELEASE=, ceph=True, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, maintainer=Guillaume Abrioux , GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240319) 2024-03-21T12:46:54.286 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:46:54 smithi189 podman[64546]: 2024-03-21 12:46:54.034163772 +0000 UTC m=+0.414541208 container start 120187b22491d4e2edea617c696f6f032b35a54a624ceb7f5c916ee9ea55d7a7 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7-deactivate, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, org.label-schema.build-date=20240319, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-21T12:46:54.286 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:46:54 smithi189 podman[64546]: 2024-03-21 12:46:54.137733233 +0000 UTC m=+0.518110672 container attach 120187b22491d4e2edea617c696f6f032b35a54a624ceb7f5c916ee9ea55d7a7 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7-deactivate, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, org.label-schema.build-date=20240319) 2024-03-21T12:46:54.338 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:54 smithi022 ceph-mon[67047]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-03-21T12:46:54.339 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:54 smithi022 ceph-mon[67047]: Health check failed: Degraded data redundancy: 16/1635 objects degraded (0.979%), 4 pgs degraded (PG_DEGRADED) 2024-03-21T12:46:54.708 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:46:54 smithi189 podman[64546]: 2024-03-21 12:46:54.392267977 +0000 UTC m=+0.772645410 container died 120187b22491d4e2edea617c696f6f032b35a54a624ceb7f5c916ee9ea55d7a7 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7-deactivate, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240319, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-21T12:46:54.891 INFO:teuthology.orchestra.run.smithi022.stdout: 5 288889 9.77 MB/sec execute 594 sec latency 33.769 ms 2024-03-21T12:46:55.343 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:55 smithi022 ceph-mon[67047]: pgmap v301: 97 pgs: 4 active+undersized+degraded, 16 peering, 8 stale+active+clean, 69 active+clean; 337 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.1 MiB/s wr, 542 op/s; 16/1635 objects degraded (0.979%) 2024-03-21T12:46:55.448 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:55 smithi189 ceph-mon[53160]: pgmap v301: 97 pgs: 4 active+undersized+degraded, 16 peering, 8 stale+active+clean, 69 active+clean; 337 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.1 MiB/s wr, 542 op/s; 16/1635 objects degraded (0.979%) 2024-03-21T12:46:55.448 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:46:55 smithi189 podman[64572]: 2024-03-21 12:46:55.183503608 +0000 UTC m=+0.784163161 container remove 120187b22491d4e2edea617c696f6f032b35a54a624ceb7f5c916ee9ea55d7a7 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7-deactivate, org.label-schema.schema-version=1.0, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.build-date=20240319) 2024-03-21T12:46:55.448 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:46:55 smithi189 systemd[1]: ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715@osd.7.service: Deactivated successfully. 2024-03-21T12:46:55.448 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:46:55 smithi189 systemd[1]: Stopped Ceph osd.7 for 3fc7f0f2-e77e-11ee-95cd-87774f69a715. 2024-03-21T12:46:55.448 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:46:55 smithi189 systemd[1]: ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715@osd.7.service: Consumed 8min 7.236s CPU time. 2024-03-21T12:46:55.700 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:46:55 smithi189 systemd[1]: Starting Ceph osd.7 for 3fc7f0f2-e77e-11ee-95cd-87774f69a715... 2024-03-21T12:46:55.701 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:46:55 smithi189 podman[64670]: 2024-03-21 12:46:55.588750008 +0000 UTC m=+0.019756999 image pull 19e12293512ff7978c8adb5f72a829060dd47f07bfd6903b6b3178d2ebc29e4e quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93 2024-03-21T12:46:55.891 INFO:teuthology.orchestra.run.smithi022.stdout: 5 289461 9.77 MB/sec execute 595 sec latency 36.359 ms 2024-03-21T12:46:55.956 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:46:55 smithi189 podman[64670]: 2024-03-21 12:46:55.701123853 +0000 UTC m=+0.132130840 container create dcbcb16aae7e45f8b7e3e67774431f6cced0f63cb953a60e0d556784c7648ddd (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7-activate, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, org.label-schema.build-date=20240319, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-03-21T12:46:55.956 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:46:55 smithi189 podman[64670]: 2024-03-21 12:46:55.955414605 +0000 UTC m=+0.386421599 container init dcbcb16aae7e45f8b7e3e67774431f6cced0f63cb953a60e0d556784c7648ddd (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7-activate, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240319, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=) 2024-03-21T12:46:56.286 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:46:55 smithi189 podman[64670]: 2024-03-21 12:46:55.959785149 +0000 UTC m=+0.390792131 container start dcbcb16aae7e45f8b7e3e67774431f6cced0f63cb953a60e0d556784c7648ddd (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7-activate, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240319, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-03-21T12:46:56.286 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:46:56 smithi189 podman[64670]: 2024-03-21 12:46:56.078749504 +0000 UTC m=+0.509756490 container attach dcbcb16aae7e45f8b7e3e67774431f6cced0f63cb953a60e0d556784c7648ddd (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7-activate, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, maintainer=Guillaume Abrioux , RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, org.label-schema.build-date=20240319) 2024-03-21T12:46:56.891 INFO:teuthology.orchestra.run.smithi022.stdout: 5 289723 9.77 MB/sec execute 596 sec latency 52.449 ms 2024-03-21T12:46:56.891 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:46:56 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7-activate[64684]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-03-21T12:46:56.891 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:46:56 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7-activate[64684]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-03-21T12:46:56.892 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:46:56 smithi189 bash[64670]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-03-21T12:46:56.892 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:46:56 smithi189 bash[64670]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-03-21T12:46:56.892 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:46:56 smithi189 bash[64670]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-03-21T12:46:56.892 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:46:56 smithi189 bash[64670]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-03-21T12:46:56.892 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:46:56 smithi189 bash[64670]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-03-21T12:46:56.892 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:46:56 smithi189 bash[64670]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-03-21T12:46:56.892 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:46:56 smithi189 bash[64670]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-03-21T12:46:56.892 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:46:56 smithi189 bash[64670]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-03-21T12:46:56.892 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:46:56 smithi189 bash[64670]: --> ceph-volume raw activate successful for osd ID: 7 2024-03-21T12:46:56.892 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:46:56 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7-activate[64684]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-03-21T12:46:56.892 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:46:56 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7-activate[64684]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-03-21T12:46:56.892 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:46:56 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7-activate[64684]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-03-21T12:46:56.892 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:46:56 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7-activate[64684]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-03-21T12:46:56.892 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:46:56 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7-activate[64684]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-03-21T12:46:56.892 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:46:56 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7-activate[64684]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-03-21T12:46:56.893 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:46:56 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7-activate[64684]: --> ceph-volume raw activate successful for osd ID: 7 2024-03-21T12:46:56.893 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:46:56 smithi189 podman[64670]: 2024-03-21 12:46:56.576468286 +0000 UTC m=+1.007475276 container died dcbcb16aae7e45f8b7e3e67774431f6cced0f63cb953a60e0d556784c7648ddd (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7-activate, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, io.buildah.version=1.33.5, org.label-schema.build-date=20240319, org.label-schema.schema-version=1.0, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-03-21T12:46:57.395 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:57 smithi022 ceph-mon[67047]: pgmap v302: 97 pgs: 2 active+undersized, 18 active+undersized+degraded, 16 peering, 1 stale+active+clean, 60 active+clean; 338 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 3.7 MiB/s wr, 372 op/s; 156/1647 objects degraded (9.472%) 2024-03-21T12:46:57.433 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:57 smithi189 ceph-mon[53160]: pgmap v302: 97 pgs: 2 active+undersized, 18 active+undersized+degraded, 16 peering, 1 stale+active+clean, 60 active+clean; 338 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 3.7 MiB/s wr, 372 op/s; 156/1647 objects degraded (9.472%) 2024-03-21T12:46:57.433 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:46:57 smithi189 podman[64823]: 2024-03-21 12:46:57.174212629 +0000 UTC m=+0.585295579 container remove dcbcb16aae7e45f8b7e3e67774431f6cced0f63cb953a60e0d556784c7648ddd (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7-activate, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.build-date=20240319, maintainer=Guillaume Abrioux , ceph=True) 2024-03-21T12:46:57.433 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:46:57 smithi189 podman[64881]: 2024-03-21 12:46:57.312815236 +0000 UTC m=+0.020183241 image pull 19e12293512ff7978c8adb5f72a829060dd47f07bfd6903b6b3178d2ebc29e4e quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93 2024-03-21T12:46:57.433 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:46:57 smithi189 podman[64881]: 2024-03-21 12:46:57.433702534 +0000 UTC m=+0.141070521 container create 247b82310a8a7c728b422e3d15a1ce8b335d605536663f694ed953d0d54bd2e3 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240319, maintainer=Guillaume Abrioux , ceph=True, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-21T12:46:57.891 INFO:teuthology.orchestra.run.smithi022.stdout: 5 290014 9.77 MB/sec execute 597 sec latency 64.053 ms 2024-03-21T12:46:58.286 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:46:57 smithi189 podman[64881]: 2024-03-21 12:46:57.919481571 +0000 UTC m=+0.626849557 container init 247b82310a8a7c728b422e3d15a1ce8b335d605536663f694ed953d0d54bd2e3 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240319, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-03-21T12:46:58.286 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:46:57 smithi189 podman[64881]: 2024-03-21 12:46:57.922964789 +0000 UTC m=+0.630332784 container start 247b82310a8a7c728b422e3d15a1ce8b335d605536663f694ed953d0d54bd2e3 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7, org.label-schema.schema-version=1.0, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, io.buildah.version=1.33.5, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240319) 2024-03-21T12:46:58.286 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:46:57 smithi189 ceph-osd[64900]: -- 172.21.15.189:0/1184891108 <== mon.0 v2:172.21.15.22:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5608e7987680 con 0x5608e7951000 2024-03-21T12:46:58.286 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:46:58 smithi189 bash[64881]: 247b82310a8a7c728b422e3d15a1ce8b335d605536663f694ed953d0d54bd2e3 2024-03-21T12:46:58.286 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:46:58 smithi189 systemd[1]: Started Ceph osd.7 for 3fc7f0f2-e77e-11ee-95cd-87774f69a715. 2024-03-21T12:46:58.786 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:46:58 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:46:58.460+0000 7fe2827cc7c0 -1 Falling back to public interface 2024-03-21T12:46:58.891 INFO:teuthology.orchestra.run.smithi022.stdout: 5 290286 9.77 MB/sec execute 598 sec latency 56.626 ms 2024-03-21T12:46:59.168 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:59 smithi022 ceph-mon[67047]: pgmap v303: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 337 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.1 MiB/s wr, 479 op/s; 265/1650 objects degraded (16.061%) 2024-03-21T12:46:59.169 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:59 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:46:59.169 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:59 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:46:59.169 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:59 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:46:59.169 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:46:59 smithi022 ceph-mon[67047]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-03-21T12:46:59.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:59 smithi189 ceph-mon[53160]: pgmap v303: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 337 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.1 MiB/s wr, 479 op/s; 265/1650 objects degraded (16.061%) 2024-03-21T12:46:59.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:59 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:46:59.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:59 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:46:59.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:59 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:46:59.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:46:59 smithi189 ceph-mon[53160]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-03-21T12:46:59.891 INFO:teuthology.orchestra.run.smithi022.stdout: 5 290674 9.77 MB/sec execute 599 sec latency 56.332 ms 2024-03-21T12:47:00.185 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:00 smithi022 ceph-mon[67047]: Health check update: Degraded data redundancy: 250/1593 objects degraded (15.694%), 35 pgs degraded (PG_DEGRADED) 2024-03-21T12:47:00.274 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:00 smithi189 ceph-mon[53160]: Health check update: Degraded data redundancy: 250/1593 objects degraded (15.694%), 35 pgs degraded (PG_DEGRADED) 2024-03-21T12:47:00.891 INFO:teuthology.orchestra.run.smithi022.stdout: 5 cleanup 600 sec 2024-03-21T12:47:01.045 INFO:teuthology.orchestra.run.smithi022.stdout:[292925] unlink './clients/client3/~dmtmp/PARADOX/.nfs00000100000002bf0000000f' failed - Device or resource busy 2024-03-21T12:47:01.045 INFO:teuthology.orchestra.run.smithi022.stdout:[292925] unlink './clients/client3/~dmtmp/PARADOX/.nfs000001000000031b0000000a' failed - Device or resource busy 2024-03-21T12:47:01.045 INFO:teuthology.orchestra.run.smithi022.stdout:[292925] unlink './clients/client3/~dmtmp/PARADOX/.nfs00000100000002cf00000010' failed - Device or resource busy 2024-03-21T12:47:01.046 INFO:teuthology.orchestra.run.smithi022.stdout:[292925] unlink './clients/client3/~dmtmp/PARADOX/.nfs00000100000002f800000012' failed - Device or resource busy 2024-03-21T12:47:01.046 INFO:teuthology.orchestra.run.smithi022.stdout:[292925] unlink './clients/client3/~dmtmp/PARADOX/.nfs00000100000002c900000005' failed - Device or resource busy 2024-03-21T12:47:01.047 INFO:teuthology.orchestra.run.smithi022.stdout:[292925] unlink './clients/client3/~dmtmp/PARADOX/.nfs00000100000002e400000006' failed - Device or resource busy 2024-03-21T12:47:01.047 INFO:teuthology.orchestra.run.smithi022.stdout:[292925] unlink './clients/client3/~dmtmp/PARADOX/.nfs000001000000030200000007' failed - Device or resource busy 2024-03-21T12:47:01.048 INFO:teuthology.orchestra.run.smithi022.stdout:[292925] unlink './clients/client3/~dmtmp/PARADOX/.nfs00000100000003210000000d' failed - Device or resource busy 2024-03-21T12:47:01.048 INFO:teuthology.orchestra.run.smithi022.stdout:[292925] unlink './clients/client3/~dmtmp/PARADOX/.nfs00000100000002f20000000e' failed - Device or resource busy 2024-03-21T12:47:01.049 INFO:teuthology.orchestra.run.smithi022.stdout:[292925] unlink './clients/client3/~dmtmp/PARADOX/.nfs000001000000031600000009' failed - Device or resource busy 2024-03-21T12:47:01.071 INFO:teuthology.orchestra.run.smithi022.stdout:[292925] unlink './clients/client3/~dmtmp/PARADOX/.nfs00000100000002c400000004' failed - Device or resource busy 2024-03-21T12:47:01.092 INFO:teuthology.orchestra.run.smithi022.stdout:[292925] unlink './clients/client3/~dmtmp/PARADOX/.nfs000001000000030c0000000c' failed - Device or resource busy 2024-03-21T12:47:01.104 INFO:teuthology.orchestra.run.smithi022.stdout:[292925] unlink './clients/client3/~dmtmp/PARADOX/.nfs00000100000002ee0000000b' failed - Device or resource busy 2024-03-21T12:47:01.107 INFO:teuthology.orchestra.run.smithi022.stdout:[292925] unlink './clients/client3/~dmtmp/PARADOX/.nfs00000100000002da00000008' failed - Device or resource busy 2024-03-21T12:47:01.108 INFO:teuthology.orchestra.run.smithi022.stdout:[292925] unlink './clients/client3/~dmtmp/PARADOX/.nfs000001000000030700000011' failed - Device or resource busy 2024-03-21T12:47:01.151 INFO:teuthology.orchestra.run.smithi022.stdout: 0 cleanup 600 sec 2024-03-21T12:47:01.151 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-03-21T12:47:01.151 INFO:teuthology.orchestra.run.smithi022.stdout: Operation Count AvgLat MaxLat 2024-03-21T12:47:01.151 INFO:teuthology.orchestra.run.smithi022.stdout: ---------------------------------------- 2024-03-21T12:47:01.151 INFO:teuthology.orchestra.run.smithi022.stdout: NTCreateX 187821 4.604 2455.812 2024-03-21T12:47:01.152 INFO:teuthology.orchestra.run.smithi022.stdout: Close 137846 0.141 13.600 2024-03-21T12:47:01.152 INFO:teuthology.orchestra.run.smithi022.stdout: Rename 7955 9.105 95.342 2024-03-21T12:47:01.152 INFO:teuthology.orchestra.run.smithi022.stdout: Unlink 38014 6.912 1073.213 2024-03-21T12:47:01.152 INFO:teuthology.orchestra.run.smithi022.stdout: Qpathinfo 170345 1.199 2416.272 2024-03-21T12:47:01.152 INFO:teuthology.orchestra.run.smithi022.stdout: Qfileinfo 29671 0.008 0.932 2024-03-21T12:47:01.152 INFO:teuthology.orchestra.run.smithi022.stdout: Qfsinfo 31246 0.673 33.159 2024-03-21T12:47:01.152 INFO:teuthology.orchestra.run.smithi022.stdout: Sfileinfo 15288 13.025 2432.070 2024-03-21T12:47:01.152 INFO:teuthology.orchestra.run.smithi022.stdout: Find 65795 1.010 14.370 2024-03-21T12:47:01.152 INFO:teuthology.orchestra.run.smithi022.stdout: WriteX 92751 13.833 2983.788 2024-03-21T12:47:01.152 INFO:teuthology.orchestra.run.smithi022.stdout: ReadX 294593 0.010 4.633 2024-03-21T12:47:01.152 INFO:teuthology.orchestra.run.smithi022.stdout: LockX 612 0.417 8.022 2024-03-21T12:47:01.152 INFO:teuthology.orchestra.run.smithi022.stdout: UnlockX 612 0.332 4.470 2024-03-21T12:47:01.152 INFO:teuthology.orchestra.run.smithi022.stdout: Flush 13160 0.002 0.021 2024-03-21T12:47:01.152 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-03-21T12:47:01.152 INFO:teuthology.orchestra.run.smithi022.stdout:Throughput 9.76542 MB/sec 5 clients 5 procs max_latency=2983.795 ms 2024-03-21T12:47:01.154 DEBUG:teuthology.orchestra.run.smithi022:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'umount /mnt/foo' 2024-03-21T12:47:01.193 DEBUG:teuthology.orchestra.run.smithi022:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'while ! mount -t nfs $(hostname):/fake /mnt/foo ; do sleep 5 ; done' 2024-03-21T12:47:01.260 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:01 smithi022 ceph-mon[67047]: pgmap v304: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 335 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.6 MiB/s wr, 516 op/s; 250/1593 objects degraded (15.694%) 2024-03-21T12:47:01.311 DEBUG:teuthology.orchestra.run.smithi022:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'cd /mnt/foo && dbench 5 -t 5' 2024-03-21T12:47:01.380 INFO:teuthology.orchestra.run.smithi022.stdout:dbench version 4.00 - Copyright Andrew Tridgell 1999-2004 2024-03-21T12:47:01.380 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-03-21T12:47:01.381 INFO:teuthology.orchestra.run.smithi022.stdout:Running for 5 seconds with load '/usr/share/dbench/client.txt' and minimum warmup 1 secs 2024-03-21T12:47:01.381 INFO:teuthology.orchestra.run.smithi022.stdout:3 of 5 processes prepared for launch 0 sec 2024-03-21T12:47:01.438 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:01 smithi189 ceph-mon[53160]: pgmap v304: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 335 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.6 MiB/s wr, 516 op/s; 250/1593 objects degraded (15.694%) 2024-03-21T12:47:01.481 INFO:teuthology.orchestra.run.smithi022.stdout:5 of 5 processes prepared for launch 0 sec 2024-03-21T12:47:01.481 INFO:teuthology.orchestra.run.smithi022.stdout:releasing clients 2024-03-21T12:47:03.338 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:03 smithi022 ceph-mon[67047]: pgmap v305: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 341 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 5.2 MiB/s wr, 554 op/s; 250/1587 objects degraded (15.753%) 2024-03-21T12:47:03.339 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:03 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:47:03.339 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:03 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:47:03.481 INFO:teuthology.orchestra.run.smithi022.stdout: 5 335 19.25 MB/sec execute 1 sec latency 56.117 ms 2024-03-21T12:47:03.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:03 smithi189 ceph-mon[53160]: pgmap v305: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 341 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 5.2 MiB/s wr, 554 op/s; 250/1587 objects degraded (15.753%) 2024-03-21T12:47:03.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:03 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:47:03.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:03 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:47:04.481 INFO:teuthology.orchestra.run.smithi022.stdout: 5 446 18.47 MB/sec execute 2 sec latency 68.707 ms 2024-03-21T12:47:05.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:04 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:47:05.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:04 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:47:05.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:04 smithi189 ceph-mon[53160]: pgmap v306: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 351 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 6.0 MiB/s wr, 446 op/s; 263/1650 objects degraded (15.939%) 2024-03-21T12:47:05.245 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:04 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:47:05.246 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:04 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:47:05.246 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:04 smithi022 ceph-mon[67047]: pgmap v306: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 351 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 6.0 MiB/s wr, 446 op/s; 263/1650 objects degraded (15.939%) 2024-03-21T12:47:05.481 INFO:teuthology.orchestra.run.smithi022.stdout: 5 665 21.45 MB/sec execute 3 sec latency 32.819 ms 2024-03-21T12:47:06.481 INFO:teuthology.orchestra.run.smithi022.stdout: 5 933 16.89 MB/sec execute 4 sec latency 37.474 ms 2024-03-21T12:47:06.794 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:06 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:47:06.794 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:06 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:47:06.794 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:06 smithi022 ceph-mon[67047]: pgmap v307: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 347 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 9.1 MiB/s wr, 445 op/s; 233/1545 objects degraded (15.081%) 2024-03-21T12:47:06.950 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:06 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:47:06.950 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:06 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:47:06.950 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:06 smithi189 ceph-mon[53160]: pgmap v307: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 347 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 9.1 MiB/s wr, 445 op/s; 233/1545 objects degraded (15.081%) 2024-03-21T12:47:07.481 INFO:teuthology.orchestra.run.smithi022.stdout: 5 cleanup 5 sec 2024-03-21T12:47:07.682 INFO:teuthology.orchestra.run.smithi022.stdout: 0 cleanup 5 sec 2024-03-21T12:47:07.683 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-03-21T12:47:07.683 INFO:teuthology.orchestra.run.smithi022.stdout: Operation Count AvgLat MaxLat 2024-03-21T12:47:07.683 INFO:teuthology.orchestra.run.smithi022.stdout: ---------------------------------------- 2024-03-21T12:47:07.683 INFO:teuthology.orchestra.run.smithi022.stdout: NTCreateX 732 9.291 68.702 2024-03-21T12:47:07.683 INFO:teuthology.orchestra.run.smithi022.stdout: Close 679 8.894 50.658 2024-03-21T12:47:07.683 INFO:teuthology.orchestra.run.smithi022.stdout: Rename 46 14.584 45.755 2024-03-21T12:47:07.683 INFO:teuthology.orchestra.run.smithi022.stdout: Unlink 71 4.162 48.602 2024-03-21T12:47:07.683 INFO:teuthology.orchestra.run.smithi022.stdout: Qpathinfo 414 1.503 26.822 2024-03-21T12:47:07.683 INFO:teuthology.orchestra.run.smithi022.stdout: Qfileinfo 336 19.345 59.209 2024-03-21T12:47:07.683 INFO:teuthology.orchestra.run.smithi022.stdout: Qfsinfo 143 0.373 2.261 2024-03-21T12:47:07.683 INFO:teuthology.orchestra.run.smithi022.stdout: Sfileinfo 148 15.099 48.043 2024-03-21T12:47:07.683 INFO:teuthology.orchestra.run.smithi022.stdout: Find 187 0.740 2.912 2024-03-21T12:47:07.683 INFO:teuthology.orchestra.run.smithi022.stdout: WriteX 1625 0.023 0.059 2024-03-21T12:47:07.684 INFO:teuthology.orchestra.run.smithi022.stdout: ReadX 641 0.019 0.536 2024-03-21T12:47:07.684 INFO:teuthology.orchestra.run.smithi022.stdout: Flush 120 13.910 25.882 2024-03-21T12:47:07.684 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-03-21T12:47:07.684 INFO:teuthology.orchestra.run.smithi022.stdout:Throughput 16.8891 MB/sec 5 clients 5 procs max_latency=68.707 ms 2024-03-21T12:47:07.685 DEBUG:teuthology.parallel:result is None 2024-03-21T12:47:07.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:07 smithi022 ceph-mon[67047]: Health check update: Degraded data redundancy: 233/1545 objects degraded (15.081%), 35 pgs degraded (PG_DEGRADED) 2024-03-21T12:47:08.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:07 smithi189 ceph-mon[53160]: Health check update: Degraded data redundancy: 233/1545 objects degraded (15.081%), 35 pgs degraded (PG_DEGRADED) 2024-03-21T12:47:08.983 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:08 smithi189 ceph-mon[53160]: pgmap v308: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 322 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 11 MiB/s wr, 564 op/s; 194/1317 objects degraded (14.730%) 2024-03-21T12:47:09.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:08 smithi022 ceph-mon[67047]: pgmap v308: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 322 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 11 MiB/s wr, 564 op/s; 194/1317 objects degraded (14.730%) 2024-03-21T12:47:10.035 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:47:09 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:47:09.610+0000 7fe2827cc7c0 -1 osd.7 0 read_superblock omap replica is missing. 2024-03-21T12:47:10.345 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:10 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:47:10.345 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:10 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:47:10.345 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:10 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:47:10.345 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:10 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:47:10.345 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:10 smithi022 ceph-mon[67047]: pgmap v309: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 315 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 11 MiB/s wr, 518 op/s; 194/1281 objects degraded (15.144%) 2024-03-21T12:47:10.345 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:10 smithi022 ceph-mon[67047]: pgmap v310: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 315 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 9.8 KiB/s rd, 13 MiB/s wr, 478 op/s; 194/1281 objects degraded (15.144%) 2024-03-21T12:47:10.345 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:10 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:47:10.345 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:10 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:47:10.345 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:10 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:47:10.346 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:10 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:47:10.346 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:10 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:47:10.346 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:10 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:47:10.346 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:10 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:47:10.346 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:10 smithi022 ceph-mon[67047]: Upgrade: Setting container_image for all osd 2024-03-21T12:47:10.346 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:10 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:47:10.346 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:10 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-03-21T12:47:10.346 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:10 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-03-21T12:47:10.346 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:10 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-03-21T12:47:10.346 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:10 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-03-21T12:47:10.346 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:10 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-03-21T12:47:10.346 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:10 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-03-21T12:47:10.346 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:10 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-03-21T12:47:10.346 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:10 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-03-21T12:47:10.346 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:10 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-03-21T12:47:10.347 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:10 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-03-21T12:47:10.347 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:10 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-03-21T12:47:10.347 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:10 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-03-21T12:47:10.347 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:10 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-03-21T12:47:10.347 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:10 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-03-21T12:47:10.347 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:10 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-03-21T12:47:10.347 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:10 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-03-21T12:47:10.347 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:10 smithi022 ceph-mon[67047]: Upgrade: Setting require_osd_release to 19 squid 2024-03-21T12:47:10.347 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:10 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-03-21T12:47:10.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:10 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:47:10.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:10 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:47:10.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:10 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:47:10.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:10 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:47:10.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:10 smithi189 ceph-mon[53160]: pgmap v309: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 315 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 11 MiB/s wr, 518 op/s; 194/1281 objects degraded (15.144%) 2024-03-21T12:47:10.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:10 smithi189 ceph-mon[53160]: pgmap v310: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 315 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 9.8 KiB/s rd, 13 MiB/s wr, 478 op/s; 194/1281 objects degraded (15.144%) 2024-03-21T12:47:10.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:10 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:47:10.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:10 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:47:10.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:10 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:47:10.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:10 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:47:10.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:10 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:47:10.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:10 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:47:10.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:10 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:47:10.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:10 smithi189 ceph-mon[53160]: Upgrade: Setting container_image for all osd 2024-03-21T12:47:10.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:10 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:47:10.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:10 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-03-21T12:47:10.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:10 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-03-21T12:47:10.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:10 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-03-21T12:47:10.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:10 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-03-21T12:47:10.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:10 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-03-21T12:47:10.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:10 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-03-21T12:47:10.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:10 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-03-21T12:47:10.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:10 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-03-21T12:47:10.538 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:10 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-03-21T12:47:10.538 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:10 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-03-21T12:47:10.538 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:10 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-03-21T12:47:10.538 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:10 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-03-21T12:47:10.538 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:10 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-03-21T12:47:10.538 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:10 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-03-21T12:47:10.538 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:10 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-03-21T12:47:10.538 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:10 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-03-21T12:47:10.538 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:10 smithi189 ceph-mon[53160]: Upgrade: Setting require_osd_release to 19 squid 2024-03-21T12:47:10.538 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:10 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-03-21T12:47:11.343 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:47:11 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:47:11.157+0000 7fe2827cc7c0 -1 osd.7 151 log_to_monitors true 2024-03-21T12:47:11.343 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:11 smithi189 ceph-mon[53160]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-03-21T12:47:11.343 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:11 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-03-21T12:47:11.343 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:11 smithi189 ceph-mon[53160]: osdmap e154: 8 total, 7 up, 8 in 2024-03-21T12:47:11.343 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:11 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi022.bcnxho"]}]: dispatch 2024-03-21T12:47:11.344 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:11 smithi189 ceph-mon[53160]: Upgrade: It appears safe to stop mds.foofs.smithi022.bcnxho 2024-03-21T12:47:11.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:11 smithi022 ceph-mon[67047]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-03-21T12:47:11.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:11 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-03-21T12:47:11.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:11 smithi022 ceph-mon[67047]: osdmap e154: 8 total, 7 up, 8 in 2024-03-21T12:47:11.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:11 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi022.bcnxho"]}]: dispatch 2024-03-21T12:47:11.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:11 smithi022 ceph-mon[67047]: Upgrade: It appears safe to stop mds.foofs.smithi022.bcnxho 2024-03-21T12:47:12.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:12 smithi022 ceph-mon[67047]: pgmap v312: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 313 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 14 MiB/s wr, 396 op/s; 177/1233 objects degraded (14.355%) 2024-03-21T12:47:12.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:12 smithi022 ceph-mon[67047]: from='osd.7 [v2:172.21.15.189:6824/4016474519,v1:172.21.15.189:6825/4016474519]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-03-21T12:47:12.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:12 smithi022 ceph-mon[67047]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-03-21T12:47:12.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:12 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:47:12.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:12 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi022.bcnxho", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-21T12:47:12.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:12 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:47:12.524 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:12 smithi189 ceph-mon[53160]: pgmap v312: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 313 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 14 MiB/s wr, 396 op/s; 177/1233 objects degraded (14.355%) 2024-03-21T12:47:12.524 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:12 smithi189 ceph-mon[53160]: from='osd.7 [v2:172.21.15.189:6824/4016474519,v1:172.21.15.189:6825/4016474519]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-03-21T12:47:12.524 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:12 smithi189 ceph-mon[53160]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-03-21T12:47:12.524 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:12 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:47:12.524 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:12 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi022.bcnxho", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-21T12:47:12.524 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:12 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:47:12.786 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:47:12 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:47:12.523+0000 7fe26fc1c700 -1 osd.7 151 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-21T12:47:13.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:13 smithi022 ceph-mon[67047]: Upgrade: Updating mds.foofs.smithi022.bcnxho 2024-03-21T12:47:13.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:13 smithi022 ceph-mon[67047]: Deploying daemon mds.foofs.smithi022.bcnxho on smithi022 2024-03-21T12:47:13.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:13 smithi022 ceph-mon[67047]: Health check update: Degraded data redundancy: 177/1233 objects degraded (14.355%), 35 pgs degraded (PG_DEGRADED) 2024-03-21T12:47:13.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:13 smithi022 ceph-mon[67047]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-03-21T12:47:13.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:13 smithi022 ceph-mon[67047]: from='osd.7 [v2:172.21.15.189:6824/4016474519,v1:172.21.15.189:6825/4016474519]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-03-21T12:47:13.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:13 smithi022 ceph-mon[67047]: osdmap e155: 8 total, 7 up, 8 in 2024-03-21T12:47:13.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:13 smithi022 ceph-mon[67047]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-03-21T12:47:13.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:13 smithi022 ceph-mon[67047]: from='osd.7 ' entity='osd.7' 2024-03-21T12:47:13.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:13 smithi189 ceph-mon[53160]: Upgrade: Updating mds.foofs.smithi022.bcnxho 2024-03-21T12:47:13.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:13 smithi189 ceph-mon[53160]: Deploying daemon mds.foofs.smithi022.bcnxho on smithi022 2024-03-21T12:47:13.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:13 smithi189 ceph-mon[53160]: Health check update: Degraded data redundancy: 177/1233 objects degraded (14.355%), 35 pgs degraded (PG_DEGRADED) 2024-03-21T12:47:13.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:13 smithi189 ceph-mon[53160]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-03-21T12:47:13.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:13 smithi189 ceph-mon[53160]: from='osd.7 [v2:172.21.15.189:6824/4016474519,v1:172.21.15.189:6825/4016474519]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-03-21T12:47:13.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:13 smithi189 ceph-mon[53160]: osdmap e155: 8 total, 7 up, 8 in 2024-03-21T12:47:13.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:13 smithi189 ceph-mon[53160]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-03-21T12:47:13.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:13 smithi189 ceph-mon[53160]: from='osd.7 ' entity='osd.7' 2024-03-21T12:47:14.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:14 smithi022 ceph-mon[67047]: pgmap v314: 97 pgs: 9 active+undersized, 33 active+undersized+degraded, 55 active+clean; 277 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.0 MiB/s wr, 155 op/s; 139/885 objects degraded (15.706%) 2024-03-21T12:47:14.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:14 smithi022 ceph-mon[67047]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-21T12:47:14.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:14 smithi022 ceph-mon[67047]: osd.7 [v2:172.21.15.189:6824/4016474519,v1:172.21.15.189:6825/4016474519] boot 2024-03-21T12:47:14.345 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:14 smithi022 ceph-mon[67047]: osdmap e156: 8 total, 8 up, 8 in 2024-03-21T12:47:14.345 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:14 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-21T12:47:14.345 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:14 smithi022 ceph-mon[67047]: osdmap e157: 8 total, 8 up, 8 in 2024-03-21T12:47:14.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:14 smithi189 ceph-mon[53160]: pgmap v314: 97 pgs: 9 active+undersized, 33 active+undersized+degraded, 55 active+clean; 277 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.0 MiB/s wr, 155 op/s; 139/885 objects degraded (15.706%) 2024-03-21T12:47:14.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:14 smithi189 ceph-mon[53160]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-21T12:47:14.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:14 smithi189 ceph-mon[53160]: osd.7 [v2:172.21.15.189:6824/4016474519,v1:172.21.15.189:6825/4016474519] boot 2024-03-21T12:47:14.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:14 smithi189 ceph-mon[53160]: osdmap e156: 8 total, 8 up, 8 in 2024-03-21T12:47:14.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:14 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-21T12:47:14.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:14 smithi189 ceph-mon[53160]: osdmap e157: 8 total, 8 up, 8 in 2024-03-21T12:47:14.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:14 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-mon-smithi022[67043]: 2024-03-21T12:47:14.345+0000 7f98d8fca700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-03-21T12:47:14.844 INFO:journalctl@ceph.osd.1.smithi022.stdout:Mar 21 12:47:14 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-1[75646]: 2024-03-21T12:47:14.407+0000 7f86a8578700 -1 bdev(0x55a2fb476000 /var/lib/ceph/osd/ceph-1/block) aio_submit retries 2 2024-03-21T12:47:14.845 INFO:journalctl@ceph.osd.1.smithi022.stdout:Mar 21 12:47:14 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-1[75646]: 2024-03-21T12:47:14.407+0000 7f86a2d6d700 -1 bdev(0x55a2fb476000 /var/lib/ceph/osd/ceph-1/block) aio_submit retries 2 2024-03-21T12:47:15.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:15 smithi022 ceph-mon[67047]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-03-21T12:47:15.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:15 smithi022 ceph-mon[67047]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-03-21T12:47:15.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:15 smithi022 ceph-mon[67047]: osdmap e158: 8 total, 8 up, 8 in 2024-03-21T12:47:15.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:15 smithi022 ceph-mon[67047]: Standby daemon mds.foofs.smithi189.lluvko assigned to filesystem foofs as rank 0 2024-03-21T12:47:15.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:15 smithi022 ceph-mon[67047]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-03-21T12:47:15.345 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:15 smithi022 ceph-mon[67047]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-03-21T12:47:15.345 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:15 smithi022 ceph-mon[67047]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-03-21T12:47:15.345 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:15 smithi022 ceph-mon[67047]: fsmap foofs:1/1 {0=foofs.smithi189.lluvko=up:replay} 2024-03-21T12:47:15.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:15 smithi189 ceph-mon[53160]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-03-21T12:47:15.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:15 smithi189 ceph-mon[53160]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-03-21T12:47:15.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:15 smithi189 ceph-mon[53160]: osdmap e158: 8 total, 8 up, 8 in 2024-03-21T12:47:15.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:15 smithi189 ceph-mon[53160]: Standby daemon mds.foofs.smithi189.lluvko assigned to filesystem foofs as rank 0 2024-03-21T12:47:15.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:15 smithi189 ceph-mon[53160]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-03-21T12:47:15.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:15 smithi189 ceph-mon[53160]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-03-21T12:47:15.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:15 smithi189 ceph-mon[53160]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-03-21T12:47:15.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:15 smithi189 ceph-mon[53160]: fsmap foofs:1/1 {0=foofs.smithi189.lluvko=up:replay} 2024-03-21T12:47:16.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:16 smithi189 ceph-mon[53160]: pgmap v318: 97 pgs: 2 remapped+peering, 7 active+undersized, 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 24 active+undersized+degraded, 61 active+clean; 237 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.0 MiB/s rd, 608 KiB/s wr, 127 op/s; 149/534 objects degraded (27.903%); 3.7 KiB/s, 18 objects/s recovering 2024-03-21T12:47:16.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:16 smithi189 ceph-mon[53160]: osdmap e159: 8 total, 8 up, 8 in 2024-03-21T12:47:16.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:16 smithi189 ceph-mon[53160]: mds.? [v2:172.21.15.189:6832/2684579462,v1:172.21.15.189:6833/2684579462] up:reconnect 2024-03-21T12:47:16.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:16 smithi189 ceph-mon[53160]: fsmap foofs:1/1 {0=foofs.smithi189.lluvko=up:reconnect} 2024-03-21T12:47:16.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:16 smithi022 ceph-mon[67047]: pgmap v318: 97 pgs: 2 remapped+peering, 7 active+undersized, 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 24 active+undersized+degraded, 61 active+clean; 237 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.0 MiB/s rd, 608 KiB/s wr, 127 op/s; 149/534 objects degraded (27.903%); 3.7 KiB/s, 18 objects/s recovering 2024-03-21T12:47:16.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:16 smithi022 ceph-mon[67047]: osdmap e159: 8 total, 8 up, 8 in 2024-03-21T12:47:16.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:16 smithi022 ceph-mon[67047]: mds.? [v2:172.21.15.189:6832/2684579462,v1:172.21.15.189:6833/2684579462] up:reconnect 2024-03-21T12:47:16.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:16 smithi022 ceph-mon[67047]: fsmap foofs:1/1 {0=foofs.smithi189.lluvko=up:reconnect} 2024-03-21T12:47:17.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:17 smithi189 ceph-mon[53160]: reconnect by client.24359 172.21.15.189:0/4097293788 after 0 2024-03-21T12:47:17.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:17 smithi189 ceph-mon[53160]: reconnect by client.14546 172.21.15.22:0/3037192481 after 0 2024-03-21T12:47:17.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:17 smithi189 ceph-mon[53160]: osdmap e160: 8 total, 8 up, 8 in 2024-03-21T12:47:17.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:17 smithi189 ceph-mon[53160]: mds.? [v2:172.21.15.189:6832/2684579462,v1:172.21.15.189:6833/2684579462] up:rejoin 2024-03-21T12:47:17.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:17 smithi189 ceph-mon[53160]: fsmap foofs:1/1 {0=foofs.smithi189.lluvko=up:rejoin} 2024-03-21T12:47:17.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:17 smithi189 ceph-mon[53160]: daemon mds.foofs.smithi189.lluvko is now active in filesystem foofs as rank 0 2024-03-21T12:47:17.807 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:17 smithi022 ceph-mon[67047]: reconnect by client.24359 172.21.15.189:0/4097293788 after 0 2024-03-21T12:47:17.807 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:17 smithi022 ceph-mon[67047]: reconnect by client.14546 172.21.15.22:0/3037192481 after 0 2024-03-21T12:47:17.807 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:17 smithi022 ceph-mon[67047]: osdmap e160: 8 total, 8 up, 8 in 2024-03-21T12:47:17.807 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:17 smithi022 ceph-mon[67047]: mds.? [v2:172.21.15.189:6832/2684579462,v1:172.21.15.189:6833/2684579462] up:rejoin 2024-03-21T12:47:17.807 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:17 smithi022 ceph-mon[67047]: fsmap foofs:1/1 {0=foofs.smithi189.lluvko=up:rejoin} 2024-03-21T12:47:17.807 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:17 smithi022 ceph-mon[67047]: daemon mds.foofs.smithi189.lluvko is now active in filesystem foofs as rank 0 2024-03-21T12:47:17.808 INFO:teuthology.orchestra.run.smithi022.stdout:true 2024-03-21T12:47:18.499 INFO:teuthology.orchestra.run.smithi022.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-21T12:47:18.499 INFO:teuthology.orchestra.run.smithi022.stdout:alertmanager.smithi022 smithi022 *:9093,9094 running (9m) 3m ago 18m 24.1M - 0.23.0 ba2b418f427c 14eb2a3ca318 2024-03-21T12:47:18.499 INFO:teuthology.orchestra.run.smithi022.stdout:crash.smithi022 smithi022 running (5m) 3m ago 18m 6660k - 19.0.0-2267-g960fbbe4 19e12293512f 356b2ddb609a 2024-03-21T12:47:18.499 INFO:teuthology.orchestra.run.smithi022.stdout:crash.smithi189 smithi189 running (5m) 16s ago 18m 6660k - 19.0.0-2267-g960fbbe4 19e12293512f 90ee837f0500 2024-03-21T12:47:18.499 INFO:teuthology.orchestra.run.smithi022.stdout:grafana.smithi022 smithi022 *:3000 error 3m ago 18m - - 2024-03-21T12:47:18.500 INFO:teuthology.orchestra.run.smithi022.stdout:mds.foofs.smithi022.bcnxho smithi022 running (12m) 3m ago 12m 38.3M - 17.2.0 e1d6a67b021e db26b67b8ee7 2024-03-21T12:47:18.500 INFO:teuthology.orchestra.run.smithi022.stdout:mds.foofs.smithi189.lluvko smithi189 running (12m) 16s ago 12m 18.0M - 17.2.0 e1d6a67b021e cf153aff1387 2024-03-21T12:47:18.500 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.smithi022.fektzs smithi022 *:8443,9283,8765 running (8m) 3m ago 19m 506M - 19.0.0-2267-g960fbbe4 19e12293512f d57cebd306b3 2024-03-21T12:47:18.500 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.smithi189.cnxzvu smithi189 *:8443,9283,8765 running (7m) 16s ago 17m 452M - 19.0.0-2267-g960fbbe4 19e12293512f 41663955c56f 2024-03-21T12:47:18.500 INFO:teuthology.orchestra.run.smithi022.stdout:mon.smithi022 smithi022 running (7m) 3m ago 20m 50.6M 2048M 19.0.0-2267-g960fbbe4 19e12293512f c620c9071b2c 2024-03-21T12:47:18.500 INFO:teuthology.orchestra.run.smithi022.stdout:mon.smithi189 smithi189 running (6m) 16s ago 17m 45.0M 2048M 19.0.0-2267-g960fbbe4 19e12293512f b353af3d8f3b 2024-03-21T12:47:18.500 INFO:teuthology.orchestra.run.smithi022.stdout:nfs.foo.0.0.smithi022.ynhbbq smithi022 *:2049 running (12m) 3m ago 12m 277M - 4.0 e1d6a67b021e 33e48e64c60b 2024-03-21T12:47:18.500 INFO:teuthology.orchestra.run.smithi022.stdout:nfs.foo.1.0.smithi189.yzsqml smithi189 *:2049 running (12m) 16s ago 12m 88.1M - 4.0 e1d6a67b021e 48fab2cbdde2 2024-03-21T12:47:18.500 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.smithi022 smithi022 *:9100 running (9m) 3m ago 18m 18.2M - 1.3.1 1dbe0e931976 33287f803f4f 2024-03-21T12:47:18.500 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.smithi189 smithi189 *:9100 running (8m) 16s ago 17m 17.4M - 1.3.1 1dbe0e931976 b51d872e438a 2024-03-21T12:47:18.500 INFO:teuthology.orchestra.run.smithi022.stdout:osd.0 smithi022 running (5m) 3m ago 17m 336M 1937M 19.0.0-2267-g960fbbe4 19e12293512f 23a85645eeea 2024-03-21T12:47:18.500 INFO:teuthology.orchestra.run.smithi022.stdout:osd.1 smithi022 running (4m) 3m ago 16m 286M 1937M 19.0.0-2267-g960fbbe4 19e12293512f 0f90e70bbf0a 2024-03-21T12:47:18.500 INFO:teuthology.orchestra.run.smithi022.stdout:osd.2 smithi022 running (3m) 3m ago 16m 214M 1937M 19.0.0-2267-g960fbbe4 19e12293512f 78a229f593f8 2024-03-21T12:47:18.500 INFO:teuthology.orchestra.run.smithi022.stdout:osd.3 smithi022 running (3m) 3m ago 15m 12.5M 1937M 19.0.0-2267-g960fbbe4 19e12293512f 61a63b67d9af 2024-03-21T12:47:18.500 INFO:teuthology.orchestra.run.smithi022.stdout:osd.4 smithi189 running (2m) 16s ago 15m 349M 2704M 19.0.0-2267-g960fbbe4 19e12293512f 382755cee998 2024-03-21T12:47:18.501 INFO:teuthology.orchestra.run.smithi022.stdout:osd.5 smithi189 running (111s) 16s ago 15m 315M 2704M 19.0.0-2267-g960fbbe4 19e12293512f d2c15ba54b4f 2024-03-21T12:47:18.501 INFO:teuthology.orchestra.run.smithi022.stdout:osd.6 smithi189 running (58s) 16s ago 14m 281M 2704M 19.0.0-2267-g960fbbe4 19e12293512f 291089fb3596 2024-03-21T12:47:18.501 INFO:teuthology.orchestra.run.smithi022.stdout:osd.7 smithi189 running (21s) 16s ago 14m 12.6M 2704M 19.0.0-2267-g960fbbe4 19e12293512f 247b82310a8a 2024-03-21T12:47:18.501 INFO:teuthology.orchestra.run.smithi022.stdout:prometheus.smithi022 smithi022 *:9095 running (7m) 3m ago 17m 73.9M - 2.33.4 514e6a882f6e 8171bbe41d2c 2024-03-21T12:47:18.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:18 smithi189 ceph-mon[53160]: pgmap v321: 97 pgs: 2 remapped+peering, 7 active+undersized, 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 19 active+undersized+degraded, 66 active+clean; 237 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 88 KiB/s wr, 33 op/s; 130/534 objects degraded (24.345%); 2.3 MiB/s, 21 objects/s recovering 2024-03-21T12:47:18.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:18 smithi189 ceph-mon[53160]: Health check update: Degraded data redundancy: 130/534 objects degraded (24.345%), 21 pgs degraded (PG_DEGRADED) 2024-03-21T12:47:18.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:18 smithi189 ceph-mon[53160]: osdmap e161: 8 total, 8 up, 8 in 2024-03-21T12:47:18.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:18 smithi189 ceph-mon[53160]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-03-21T12:47:18.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:18 smithi189 ceph-mon[53160]: mds.? [v2:172.21.15.189:6832/2684579462,v1:172.21.15.189:6833/2684579462] up:active 2024-03-21T12:47:18.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:18 smithi189 ceph-mon[53160]: fsmap foofs:1 {0=foofs.smithi189.lluvko=up:active} 2024-03-21T12:47:18.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:18 smithi022 ceph-mon[67047]: pgmap v321: 97 pgs: 2 remapped+peering, 7 active+undersized, 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 19 active+undersized+degraded, 66 active+clean; 237 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 88 KiB/s wr, 33 op/s; 130/534 objects degraded (24.345%); 2.3 MiB/s, 21 objects/s recovering 2024-03-21T12:47:18.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:18 smithi022 ceph-mon[67047]: Health check update: Degraded data redundancy: 130/534 objects degraded (24.345%), 21 pgs degraded (PG_DEGRADED) 2024-03-21T12:47:18.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:18 smithi022 ceph-mon[67047]: osdmap e161: 8 total, 8 up, 8 in 2024-03-21T12:47:18.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:18 smithi022 ceph-mon[67047]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-03-21T12:47:18.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:18 smithi022 ceph-mon[67047]: mds.? [v2:172.21.15.189:6832/2684579462,v1:172.21.15.189:6833/2684579462] up:active 2024-03-21T12:47:18.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:18 smithi022 ceph-mon[67047]: fsmap foofs:1 {0=foofs.smithi189.lluvko=up:active} 2024-03-21T12:47:18.908 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-03-21T12:47:18.908 INFO:teuthology.orchestra.run.smithi022.stdout: "mon": { 2024-03-21T12:47:18.908 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 2 2024-03-21T12:47:18.908 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:47:18.908 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr": { 2024-03-21T12:47:18.908 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 2 2024-03-21T12:47:18.908 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:47:18.908 INFO:teuthology.orchestra.run.smithi022.stdout: "osd": { 2024-03-21T12:47:18.908 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 8 2024-03-21T12:47:18.909 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:47:18.909 INFO:teuthology.orchestra.run.smithi022.stdout: "mds": { 2024-03-21T12:47:18.909 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1 2024-03-21T12:47:18.909 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:47:18.909 INFO:teuthology.orchestra.run.smithi022.stdout: "overall": { 2024-03-21T12:47:18.909 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-03-21T12:47:18.909 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 12 2024-03-21T12:47:18.909 INFO:teuthology.orchestra.run.smithi022.stdout: } 2024-03-21T12:47:18.909 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-03-21T12:47:19.282 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-03-21T12:47:19.282 INFO:teuthology.orchestra.run.smithi022.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93", 2024-03-21T12:47:19.282 INFO:teuthology.orchestra.run.smithi022.stdout: "in_progress": true, 2024-03-21T12:47:19.282 INFO:teuthology.orchestra.run.smithi022.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-21T12:47:19.282 INFO:teuthology.orchestra.run.smithi022.stdout: "services_complete": [ 2024-03-21T12:47:19.282 INFO:teuthology.orchestra.run.smithi022.stdout: "osd", 2024-03-21T12:47:19.283 INFO:teuthology.orchestra.run.smithi022.stdout: "mon", 2024-03-21T12:47:19.283 INFO:teuthology.orchestra.run.smithi022.stdout: "crash", 2024-03-21T12:47:19.283 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr" 2024-03-21T12:47:19.283 INFO:teuthology.orchestra.run.smithi022.stdout: ], 2024-03-21T12:47:19.283 INFO:teuthology.orchestra.run.smithi022.stdout: "progress": "14/23 daemons upgraded", 2024-03-21T12:47:19.283 INFO:teuthology.orchestra.run.smithi022.stdout: "message": "Currently upgrading mds daemons", 2024-03-21T12:47:19.283 INFO:teuthology.orchestra.run.smithi022.stdout: "is_paused": false 2024-03-21T12:47:19.283 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-03-21T12:47:19.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:19 smithi022 ceph-mon[67047]: from='client.34468 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:47:19.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:19 smithi022 ceph-mon[67047]: from='client.34472 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:47:19.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:19 smithi022 ceph-mon[67047]: from='client.? 172.21.15.22:0/1967590816' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:47:19.704 INFO:teuthology.orchestra.run.smithi022.stdout:HEALTH_WARN 1 failed cephadm daemon(s); insufficient standby MDS daemons available; Degraded data redundancy: 32/480 objects degraded (6.667%), 1 pg degraded 2024-03-21T12:47:19.704 INFO:teuthology.orchestra.run.smithi022.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-21T12:47:19.704 INFO:teuthology.orchestra.run.smithi022.stdout: daemon grafana.smithi022 on smithi022 is in error state 2024-03-21T12:47:19.704 INFO:teuthology.orchestra.run.smithi022.stdout:[WRN] MDS_INSUFFICIENT_STANDBY: insufficient standby MDS daemons available 2024-03-21T12:47:19.704 INFO:teuthology.orchestra.run.smithi022.stdout: have 0; want 1 more 2024-03-21T12:47:19.704 INFO:teuthology.orchestra.run.smithi022.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 32/480 objects degraded (6.667%), 1 pg degraded 2024-03-21T12:47:19.704 INFO:teuthology.orchestra.run.smithi022.stdout: pg 3.9 is active+recovery_wait+undersized+degraded+remapped, acting [4,2] 2024-03-21T12:47:19.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:19 smithi189 ceph-mon[53160]: from='client.34468 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:47:19.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:19 smithi189 ceph-mon[53160]: from='client.34472 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:47:19.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:19 smithi189 ceph-mon[53160]: from='client.? 172.21.15.22:0/1967590816' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:47:20.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:20 smithi189 ceph-mon[53160]: from='client.34476 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:47:20.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:20 smithi189 ceph-mon[53160]: pgmap v323: 97 pgs: 1 peering, 1 remapped+peering, 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 36 MiB/s rd, 66 KiB/s wr, 22 op/s; 32/480 objects degraded (6.667%); 1.9 MiB/s, 70 objects/s recovering 2024-03-21T12:47:20.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:20 smithi189 ceph-mon[53160]: from='client.34484 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:47:20.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:20 smithi189 ceph-mon[53160]: from='client.? 172.21.15.22:0/1678732518' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-21T12:47:20.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:20 smithi022 ceph-mon[67047]: from='client.34476 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:47:20.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:20 smithi022 ceph-mon[67047]: pgmap v323: 97 pgs: 1 peering, 1 remapped+peering, 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 36 MiB/s rd, 66 KiB/s wr, 22 op/s; 32/480 objects degraded (6.667%); 1.9 MiB/s, 70 objects/s recovering 2024-03-21T12:47:20.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:20 smithi022 ceph-mon[67047]: from='client.34484 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:47:20.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:20 smithi022 ceph-mon[67047]: from='client.? 172.21.15.22:0/1678732518' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-21T12:47:22.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:21 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:47:22.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:21 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:47:22.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:21 smithi189 ceph-mon[53160]: pgmap v324: 97 pgs: 1 peering, 96 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 32 MiB/s rd, 52 KiB/s wr, 16 op/s; 2.2 MiB/s, 53 objects/s recovering 2024-03-21T12:47:22.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:21 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:47:22.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:21 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:47:22.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:21 smithi022 ceph-mon[67047]: pgmap v324: 97 pgs: 1 peering, 96 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 32 MiB/s rd, 52 KiB/s wr, 16 op/s; 2.2 MiB/s, 53 objects/s recovering 2024-03-21T12:47:23.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:22 smithi189 ceph-mon[53160]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 32/480 objects degraded (6.667%), 1 pg degraded) 2024-03-21T12:47:23.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:22 smithi022 ceph-mon[67047]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 32/480 objects degraded (6.667%), 1 pg degraded) 2024-03-21T12:47:24.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:23 smithi189 ceph-mon[53160]: pgmap v325: 97 pgs: 97 active+clean; 224 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 25 MiB/s rd, 52 KiB/s wr, 22 op/s; 1.9 MiB/s, 46 objects/s recovering 2024-03-21T12:47:24.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:23 smithi022 ceph-mon[67047]: pgmap v325: 97 pgs: 97 active+clean; 224 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 25 MiB/s rd, 52 KiB/s wr, 22 op/s; 1.9 MiB/s, 46 objects/s recovering 2024-03-21T12:47:26.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:26 smithi022 ceph-mon[67047]: pgmap v326: 97 pgs: 97 active+clean; 224 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 46 KiB/s wr, 20 op/s; 1.7 MiB/s, 40 objects/s recovering 2024-03-21T12:47:26.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:26 smithi189 ceph-mon[53160]: pgmap v326: 97 pgs: 97 active+clean; 224 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 46 KiB/s wr, 20 op/s; 1.7 MiB/s, 40 objects/s recovering 2024-03-21T12:47:27.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:27 smithi022 ceph-mon[67047]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-03-21T12:47:27.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:27 smithi022 ceph-mon[67047]: mds.? [v2:172.21.15.22:6834/1911267561,v1:172.21.15.22:6835/1911267561] up:boot 2024-03-21T12:47:27.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:27 smithi022 ceph-mon[67047]: fsmap foofs:1 {0=foofs.smithi189.lluvko=up:active} 1 up:standby 2024-03-21T12:47:27.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:27 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi022.bcnxho"}]: dispatch 2024-03-21T12:47:27.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:27 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:47:27.345 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:27 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:47:27.345 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:27 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:47:27.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:27 smithi189 ceph-mon[53160]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-03-21T12:47:27.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:27 smithi189 ceph-mon[53160]: mds.? [v2:172.21.15.22:6834/1911267561,v1:172.21.15.22:6835/1911267561] up:boot 2024-03-21T12:47:27.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:27 smithi189 ceph-mon[53160]: fsmap foofs:1 {0=foofs.smithi189.lluvko=up:active} 1 up:standby 2024-03-21T12:47:27.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:27 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi022.bcnxho"}]: dispatch 2024-03-21T12:47:27.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:27 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:47:27.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:27 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:47:27.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:27 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:47:28.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:28 smithi189 ceph-mon[53160]: pgmap v327: 97 pgs: 97 active+clean; 224 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 11 KiB/s wr, 15 op/s; 720 KiB/s, 35 objects/s recovering 2024-03-21T12:47:28.595 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:28 smithi022 ceph-mon[67047]: pgmap v327: 97 pgs: 97 active+clean; 224 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 11 KiB/s wr, 15 op/s; 720 KiB/s, 35 objects/s recovering 2024-03-21T12:47:30.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:30 smithi022 ceph-mon[67047]: pgmap v328: 97 pgs: 97 active+clean; 224 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 8.0 MiB/s rd, 9.1 KiB/s wr, 11 op/s; 525 KiB/s, 11 objects/s recovering 2024-03-21T12:47:30.538 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:30 smithi189 ceph-mon[53160]: pgmap v328: 97 pgs: 97 active+clean; 224 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 8.0 MiB/s rd, 9.1 KiB/s wr, 11 op/s; 525 KiB/s, 11 objects/s recovering 2024-03-21T12:47:32.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:32 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:47:32.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:32 smithi022 ceph-mon[67047]: pgmap v329: 97 pgs: 97 active+clean; 224 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 8.7 KiB/s wr, 8 op/s; 512 KiB/s, 5 objects/s recovering 2024-03-21T12:47:32.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:32 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:47:32.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:32 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:47:32.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:32 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:47:32.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:32 smithi189 ceph-mon[53160]: pgmap v329: 97 pgs: 97 active+clean; 224 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 8.7 KiB/s wr, 8 op/s; 512 KiB/s, 5 objects/s recovering 2024-03-21T12:47:32.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:32 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:47:32.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:32 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:47:33.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:33 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:47:33.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:33 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:47:33.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:33 smithi022 ceph-mon[67047]: pgmap v330: 97 pgs: 97 active+clean; 224 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3.3 KiB/s rd, 8.9 KiB/s wr, 7 op/s; 171 KiB/s, 2 objects/s recovering 2024-03-21T12:47:34.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:33 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:47:34.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:33 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:47:34.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:33 smithi189 ceph-mon[53160]: pgmap v330: 97 pgs: 97 active+clean; 224 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3.3 KiB/s rd, 8.9 KiB/s wr, 7 op/s; 171 KiB/s, 2 objects/s recovering 2024-03-21T12:47:36.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:36 smithi189 ceph-mon[53160]: pgmap v331: 97 pgs: 97 active+clean; 224 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 1.4 KiB/s wr, 0 op/s 2024-03-21T12:47:36.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:36 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:47:36.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:36 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:47:36.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:36 smithi022 ceph-mon[67047]: pgmap v331: 97 pgs: 97 active+clean; 224 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 1.4 KiB/s wr, 0 op/s 2024-03-21T12:47:36.595 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:36 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:47:36.595 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:36 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:47:38.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:37 smithi189 ceph-mon[53160]: Detected new or changed devices on smithi022 2024-03-21T12:47:38.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:37 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:47:38.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:37 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:47:38.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:37 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:47:38.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:37 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:47:38.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:37 smithi189 ceph-mon[53160]: pgmap v332: 97 pgs: 97 active+clean; 224 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 175 B/s rd, 1.5 KiB/s wr, 0 op/s 2024-03-21T12:47:38.037 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:37 smithi189 ceph-mon[53160]: pgmap v333: 97 pgs: 97 active+clean; 224 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 211 B/s rd, 211 B/s wr, 0 op/s 2024-03-21T12:47:38.037 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:37 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:47:38.037 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:37 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:47:38.037 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:37 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:47:38.037 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:37 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:47:38.037 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:37 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:47:38.037 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:37 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:47:38.037 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:37 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:47:38.037 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:37 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi189.lluvko"]}]: dispatch 2024-03-21T12:47:38.037 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:37 smithi189 ceph-mon[53160]: Upgrade: It appears safe to stop mds.foofs.smithi189.lluvko 2024-03-21T12:47:38.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:37 smithi022 ceph-mon[67047]: Detected new or changed devices on smithi022 2024-03-21T12:47:38.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:37 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:47:38.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:37 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:47:38.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:37 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:47:38.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:37 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:47:38.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:37 smithi022 ceph-mon[67047]: pgmap v332: 97 pgs: 97 active+clean; 224 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 175 B/s rd, 1.5 KiB/s wr, 0 op/s 2024-03-21T12:47:38.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:37 smithi022 ceph-mon[67047]: pgmap v333: 97 pgs: 97 active+clean; 224 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 211 B/s rd, 211 B/s wr, 0 op/s 2024-03-21T12:47:38.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:37 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:47:38.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:37 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:47:38.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:37 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:47:38.096 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:37 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:47:38.096 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:37 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:47:38.096 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:37 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:47:38.096 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:37 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:47:38.096 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:37 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi189.lluvko"]}]: dispatch 2024-03-21T12:47:38.096 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:37 smithi022 ceph-mon[67047]: Upgrade: It appears safe to stop mds.foofs.smithi189.lluvko 2024-03-21T12:47:39.787 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:39 smithi189 ceph-mon[53160]: Upgrade: Updating mds.foofs.smithi189.lluvko 2024-03-21T12:47:39.787 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:39 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:47:39.787 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:39 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi189.lluvko", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-21T12:47:39.787 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:39 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:47:39.787 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:39 smithi189 ceph-mon[53160]: Deploying daemon mds.foofs.smithi189.lluvko on smithi189 2024-03-21T12:47:39.787 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:39 smithi189 ceph-mon[53160]: pgmap v334: 97 pgs: 97 active+clean; 224 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 211 B/s rd, 211 B/s wr, 0 op/s 2024-03-21T12:47:39.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:39 smithi022 ceph-mon[67047]: Upgrade: Updating mds.foofs.smithi189.lluvko 2024-03-21T12:47:39.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:39 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:47:39.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:39 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi189.lluvko", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-21T12:47:39.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:39 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:47:39.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:39 smithi022 ceph-mon[67047]: Deploying daemon mds.foofs.smithi189.lluvko on smithi189 2024-03-21T12:47:39.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:39 smithi022 ceph-mon[67047]: pgmap v334: 97 pgs: 97 active+clean; 224 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 211 B/s rd, 211 B/s wr, 0 op/s 2024-03-21T12:47:41.286 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:47:40 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-5[60346]: 2024-03-21T12:47:40.884+0000 7f06108aa700 -1 bdev(0x559d2be78000 /var/lib/ceph/osd/ceph-5/block) aio_submit retries 5 2024-03-21T12:47:41.286 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:47:40 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-5[60346]: 2024-03-21T12:47:40.885+0000 7f060f8a8700 -1 bdev(0x559d2be78000 /var/lib/ceph/osd/ceph-5/block) aio_submit retries 4 2024-03-21T12:47:41.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:40 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-mon-smithi022[67043]: 2024-03-21T12:47:40.857+0000 7f98d8fca700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-03-21T12:47:41.344 INFO:journalctl@ceph.osd.1.smithi022.stdout:Mar 21 12:47:40 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-1[75646]: 2024-03-21T12:47:40.881+0000 7f86a6d75700 -1 bdev(0x55a2fb476000 /var/lib/ceph/osd/ceph-1/block) aio_submit retries 1 2024-03-21T12:47:41.345 INFO:journalctl@ceph.osd.1.smithi022.stdout:Mar 21 12:47:40 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-1[75646]: 2024-03-21T12:47:40.881+0000 7f86a2d6d700 -1 bdev(0x55a2fb476000 /var/lib/ceph/osd/ceph-1/block) aio_submit retries 4 2024-03-21T12:47:41.345 INFO:journalctl@ceph.osd.1.smithi022.stdout:Mar 21 12:47:40 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-1[75646]: 2024-03-21T12:47:40.881+0000 7f86a4570700 -1 bdev(0x55a2fb476000 /var/lib/ceph/osd/ceph-1/block) aio_submit retries 3 2024-03-21T12:47:42.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:41 smithi022 ceph-mon[67047]: pgmap v335: 97 pgs: 97 active+clean; 224 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 211 B/s rd, 211 B/s wr, 0 op/s 2024-03-21T12:47:42.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:41 smithi022 ceph-mon[67047]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-03-21T12:47:42.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:41 smithi022 ceph-mon[67047]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-03-21T12:47:42.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:41 smithi022 ceph-mon[67047]: osdmap e162: 8 total, 8 up, 8 in 2024-03-21T12:47:42.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:41 smithi022 ceph-mon[67047]: Standby daemon mds.foofs.smithi022.bcnxho assigned to filesystem foofs as rank 0 2024-03-21T12:47:42.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:41 smithi022 ceph-mon[67047]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-03-21T12:47:42.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:41 smithi022 ceph-mon[67047]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-03-21T12:47:42.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:41 smithi022 ceph-mon[67047]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-03-21T12:47:42.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:41 smithi022 ceph-mon[67047]: fsmap foofs:1/1 {0=foofs.smithi022.bcnxho=up:replay} 2024-03-21T12:47:42.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:41 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi189.lluvko"}]: dispatch 2024-03-21T12:47:42.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:41 smithi189 ceph-mon[53160]: pgmap v335: 97 pgs: 97 active+clean; 224 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 211 B/s rd, 211 B/s wr, 0 op/s 2024-03-21T12:47:42.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:41 smithi189 ceph-mon[53160]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-03-21T12:47:42.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:41 smithi189 ceph-mon[53160]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-03-21T12:47:42.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:41 smithi189 ceph-mon[53160]: osdmap e162: 8 total, 8 up, 8 in 2024-03-21T12:47:42.287 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:41 smithi189 ceph-mon[53160]: Standby daemon mds.foofs.smithi022.bcnxho assigned to filesystem foofs as rank 0 2024-03-21T12:47:42.287 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:41 smithi189 ceph-mon[53160]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-03-21T12:47:42.287 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:41 smithi189 ceph-mon[53160]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-03-21T12:47:42.287 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:41 smithi189 ceph-mon[53160]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-03-21T12:47:42.287 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:41 smithi189 ceph-mon[53160]: fsmap foofs:1/1 {0=foofs.smithi022.bcnxho=up:replay} 2024-03-21T12:47:42.287 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:41 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi189.lluvko"}]: dispatch 2024-03-21T12:47:43.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:42 smithi189 ceph-mon[53160]: reconnect by client.14546 172.21.15.22:0/3037192481 after 0 2024-03-21T12:47:43.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:42 smithi189 ceph-mon[53160]: reconnect by client.24359 172.21.15.189:0/4097293788 after 0 2024-03-21T12:47:43.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:42 smithi189 ceph-mon[53160]: mds.? [v2:172.21.15.22:6834/1911267561,v1:172.21.15.22:6835/1911267561] up:reconnect 2024-03-21T12:47:43.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:42 smithi189 ceph-mon[53160]: fsmap foofs:1/1 {0=foofs.smithi022.bcnxho=up:reconnect} 2024-03-21T12:47:43.345 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:42 smithi022 ceph-mon[67047]: reconnect by client.14546 172.21.15.22:0/3037192481 after 0 2024-03-21T12:47:43.345 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:42 smithi022 ceph-mon[67047]: reconnect by client.24359 172.21.15.189:0/4097293788 after 0 2024-03-21T12:47:43.345 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:42 smithi022 ceph-mon[67047]: mds.? [v2:172.21.15.22:6834/1911267561,v1:172.21.15.22:6835/1911267561] up:reconnect 2024-03-21T12:47:43.345 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:42 smithi022 ceph-mon[67047]: fsmap foofs:1/1 {0=foofs.smithi022.bcnxho=up:reconnect} 2024-03-21T12:47:44.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:43 smithi189 ceph-mon[53160]: pgmap v337: 97 pgs: 97 active+clean; 224 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 5 op/s 2024-03-21T12:47:44.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:43 smithi189 ceph-mon[53160]: mds.? [v2:172.21.15.22:6834/1911267561,v1:172.21.15.22:6835/1911267561] up:rejoin 2024-03-21T12:47:44.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:43 smithi189 ceph-mon[53160]: fsmap foofs:1/1 {0=foofs.smithi022.bcnxho=up:rejoin} 2024-03-21T12:47:44.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:43 smithi189 ceph-mon[53160]: daemon mds.foofs.smithi022.bcnxho is now active in filesystem foofs as rank 0 2024-03-21T12:47:44.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:43 smithi022 ceph-mon[67047]: pgmap v337: 97 pgs: 97 active+clean; 224 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 5 op/s 2024-03-21T12:47:44.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:43 smithi022 ceph-mon[67047]: mds.? [v2:172.21.15.22:6834/1911267561,v1:172.21.15.22:6835/1911267561] up:rejoin 2024-03-21T12:47:44.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:43 smithi022 ceph-mon[67047]: fsmap foofs:1/1 {0=foofs.smithi022.bcnxho=up:rejoin} 2024-03-21T12:47:44.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:43 smithi022 ceph-mon[67047]: daemon mds.foofs.smithi022.bcnxho is now active in filesystem foofs as rank 0 2024-03-21T12:47:45.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:44 smithi189 ceph-mon[53160]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-03-21T12:47:45.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:44 smithi189 ceph-mon[53160]: mds.? [v2:172.21.15.22:6834/1911267561,v1:172.21.15.22:6835/1911267561] up:active 2024-03-21T12:47:45.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:44 smithi189 ceph-mon[53160]: fsmap foofs:1 {0=foofs.smithi022.bcnxho=up:active} 2024-03-21T12:47:45.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:44 smithi022 ceph-mon[67047]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-03-21T12:47:45.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:44 smithi022 ceph-mon[67047]: mds.? [v2:172.21.15.22:6834/1911267561,v1:172.21.15.22:6835/1911267561] up:active 2024-03-21T12:47:45.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:44 smithi022 ceph-mon[67047]: fsmap foofs:1 {0=foofs.smithi022.bcnxho=up:active} 2024-03-21T12:47:46.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:45 smithi189 ceph-mon[53160]: pgmap v338: 97 pgs: 97 active+clean; 224 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 26 MiB/s rd, 8 op/s 2024-03-21T12:47:46.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:45 smithi022 ceph-mon[67047]: pgmap v338: 97 pgs: 97 active+clean; 224 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 26 MiB/s rd, 8 op/s 2024-03-21T12:47:48.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:47 smithi189 ceph-mon[53160]: pgmap v339: 97 pgs: 97 active+clean; 224 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 7 op/s 2024-03-21T12:47:48.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:47 smithi022 ceph-mon[67047]: pgmap v339: 97 pgs: 97 active+clean; 224 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 7 op/s 2024-03-21T12:47:50.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:49 smithi189 ceph-mon[53160]: pgmap v340: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 7.3 KiB/s wr, 14 op/s 2024-03-21T12:47:50.118 INFO:teuthology.orchestra.run.smithi022.stdout:true 2024-03-21T12:47:50.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:49 smithi022 ceph-mon[67047]: pgmap v340: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 7.3 KiB/s wr, 14 op/s 2024-03-21T12:47:50.903 INFO:teuthology.orchestra.run.smithi022.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-21T12:47:50.904 INFO:teuthology.orchestra.run.smithi022.stdout:alertmanager.smithi022 smithi022 *:9093,9094 running (10m) 19s ago 19m 24.5M - 0.23.0 ba2b418f427c 14eb2a3ca318 2024-03-21T12:47:50.904 INFO:teuthology.orchestra.run.smithi022.stdout:crash.smithi022 smithi022 running (6m) 19s ago 19m 6660k - 19.0.0-2267-g960fbbe4 19e12293512f 356b2ddb609a 2024-03-21T12:47:50.904 INFO:teuthology.orchestra.run.smithi022.stdout:crash.smithi189 smithi189 running (6m) 48s ago 18m 6660k - 19.0.0-2267-g960fbbe4 19e12293512f 90ee837f0500 2024-03-21T12:47:50.904 INFO:teuthology.orchestra.run.smithi022.stdout:grafana.smithi022 smithi022 *:3000 error 19s ago 18m - - 2024-03-21T12:47:50.904 INFO:teuthology.orchestra.run.smithi022.stdout:mds.foofs.smithi022.bcnxho smithi022 running (25s) 19s ago 13m 12.2M - 19.0.0-2267-g960fbbe4 19e12293512f 6db0e27a3822 2024-03-21T12:47:50.905 INFO:teuthology.orchestra.run.smithi022.stdout:mds.foofs.smithi189.lluvko smithi189 starting - - - - 2024-03-21T12:47:50.905 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.smithi022.fektzs smithi022 *:8443,9283,8765 running (9m) 19s ago 20m 510M - 19.0.0-2267-g960fbbe4 19e12293512f d57cebd306b3 2024-03-21T12:47:50.905 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.smithi189.cnxzvu smithi189 *:8443,9283,8765 running (8m) 48s ago 18m 452M - 19.0.0-2267-g960fbbe4 19e12293512f 41663955c56f 2024-03-21T12:47:50.905 INFO:teuthology.orchestra.run.smithi022.stdout:mon.smithi022 smithi022 running (7m) 19s ago 20m 60.5M 2048M 19.0.0-2267-g960fbbe4 19e12293512f c620c9071b2c 2024-03-21T12:47:50.905 INFO:teuthology.orchestra.run.smithi022.stdout:mon.smithi189 smithi189 running (7m) 48s ago 18m 45.0M 2048M 19.0.0-2267-g960fbbe4 19e12293512f b353af3d8f3b 2024-03-21T12:47:50.906 INFO:teuthology.orchestra.run.smithi022.stdout:nfs.foo.0.0.smithi022.ynhbbq smithi022 *:2049 running (13m) 19s ago 13m 272M - 4.0 e1d6a67b021e 33e48e64c60b 2024-03-21T12:47:50.906 INFO:teuthology.orchestra.run.smithi022.stdout:nfs.foo.1.0.smithi189.yzsqml smithi189 *:2049 running (12m) 48s ago 12m 88.1M - 4.0 e1d6a67b021e 48fab2cbdde2 2024-03-21T12:47:50.906 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.smithi022 smithi022 *:9100 running (9m) 19s ago 18m 16.3M - 1.3.1 1dbe0e931976 33287f803f4f 2024-03-21T12:47:50.906 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.smithi189 smithi189 *:9100 running (9m) 48s ago 18m 17.4M - 1.3.1 1dbe0e931976 b51d872e438a 2024-03-21T12:47:50.906 INFO:teuthology.orchestra.run.smithi022.stdout:osd.0 smithi022 running (5m) 19s ago 17m 405M 1937M 19.0.0-2267-g960fbbe4 19e12293512f 23a85645eeea 2024-03-21T12:47:50.906 INFO:teuthology.orchestra.run.smithi022.stdout:osd.1 smithi022 running (5m) 19s ago 17m 382M 1937M 19.0.0-2267-g960fbbe4 19e12293512f 0f90e70bbf0a 2024-03-21T12:47:50.906 INFO:teuthology.orchestra.run.smithi022.stdout:osd.2 smithi022 running (4m) 19s ago 16m 355M 1937M 19.0.0-2267-g960fbbe4 19e12293512f 78a229f593f8 2024-03-21T12:47:50.906 INFO:teuthology.orchestra.run.smithi022.stdout:osd.3 smithi022 running (3m) 19s ago 16m 439M 1937M 19.0.0-2267-g960fbbe4 19e12293512f 61a63b67d9af 2024-03-21T12:47:50.906 INFO:teuthology.orchestra.run.smithi022.stdout:osd.4 smithi189 running (3m) 48s ago 16m 349M 2704M 19.0.0-2267-g960fbbe4 19e12293512f 382755cee998 2024-03-21T12:47:50.907 INFO:teuthology.orchestra.run.smithi022.stdout:osd.5 smithi189 running (2m) 48s ago 15m 315M 2704M 19.0.0-2267-g960fbbe4 19e12293512f d2c15ba54b4f 2024-03-21T12:47:50.907 INFO:teuthology.orchestra.run.smithi022.stdout:osd.6 smithi189 running (91s) 48s ago 15m 281M 2704M 19.0.0-2267-g960fbbe4 19e12293512f 291089fb3596 2024-03-21T12:47:50.907 INFO:teuthology.orchestra.run.smithi022.stdout:osd.7 smithi189 running (53s) 48s ago 14m 12.6M 2704M 19.0.0-2267-g960fbbe4 19e12293512f 247b82310a8a 2024-03-21T12:47:50.907 INFO:teuthology.orchestra.run.smithi022.stdout:prometheus.smithi022 smithi022 *:9095 running (8m) 19s ago 18m 77.1M - 2.33.4 514e6a882f6e 8171bbe41d2c 2024-03-21T12:47:51.009 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:50 smithi189 ceph-mon[53160]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-03-21T12:47:51.009 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:50 smithi189 ceph-mon[53160]: mds.? [v2:172.21.15.189:6832/940060883,v1:172.21.15.189:6833/940060883] up:boot 2024-03-21T12:47:51.010 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:50 smithi189 ceph-mon[53160]: fsmap foofs:1 {0=foofs.smithi022.bcnxho=up:active} 1 up:standby 2024-03-21T12:47:51.010 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:50 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi189.lluvko"}]: dispatch 2024-03-21T12:47:51.010 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:50 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:47:51.010 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:50 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:47:51.010 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:50 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:47:51.010 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:50 smithi189 ceph-mon[53160]: from='client.34496 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:47:51.010 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:50 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:47:51.344 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-03-21T12:47:51.344 INFO:teuthology.orchestra.run.smithi022.stdout: "mon": { 2024-03-21T12:47:51.344 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 2 2024-03-21T12:47:51.344 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:47:51.344 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr": { 2024-03-21T12:47:51.345 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 2 2024-03-21T12:47:51.345 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:47:51.345 INFO:teuthology.orchestra.run.smithi022.stdout: "osd": { 2024-03-21T12:47:51.345 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 8 2024-03-21T12:47:51.345 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:47:51.345 INFO:teuthology.orchestra.run.smithi022.stdout: "mds": { 2024-03-21T12:47:51.345 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 2 2024-03-21T12:47:51.345 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:47:51.345 INFO:teuthology.orchestra.run.smithi022.stdout: "overall": { 2024-03-21T12:47:51.345 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 14 2024-03-21T12:47:51.345 INFO:teuthology.orchestra.run.smithi022.stdout: } 2024-03-21T12:47:51.345 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-03-21T12:47:51.345 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:50 smithi022 ceph-mon[67047]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-03-21T12:47:51.346 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:50 smithi022 ceph-mon[67047]: mds.? [v2:172.21.15.189:6832/940060883,v1:172.21.15.189:6833/940060883] up:boot 2024-03-21T12:47:51.346 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:50 smithi022 ceph-mon[67047]: fsmap foofs:1 {0=foofs.smithi022.bcnxho=up:active} 1 up:standby 2024-03-21T12:47:51.346 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:50 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi189.lluvko"}]: dispatch 2024-03-21T12:47:51.346 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:50 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:47:51.346 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:50 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:47:51.346 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:50 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:47:51.346 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:50 smithi022 ceph-mon[67047]: from='client.34496 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:47:51.346 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:50 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:47:51.743 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-03-21T12:47:51.743 INFO:teuthology.orchestra.run.smithi022.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93", 2024-03-21T12:47:51.743 INFO:teuthology.orchestra.run.smithi022.stdout: "in_progress": true, 2024-03-21T12:47:51.743 INFO:teuthology.orchestra.run.smithi022.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-21T12:47:51.743 INFO:teuthology.orchestra.run.smithi022.stdout: "services_complete": [ 2024-03-21T12:47:51.743 INFO:teuthology.orchestra.run.smithi022.stdout: "osd", 2024-03-21T12:47:51.743 INFO:teuthology.orchestra.run.smithi022.stdout: "mon", 2024-03-21T12:47:51.744 INFO:teuthology.orchestra.run.smithi022.stdout: "crash", 2024-03-21T12:47:51.744 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr" 2024-03-21T12:47:51.744 INFO:teuthology.orchestra.run.smithi022.stdout: ], 2024-03-21T12:47:51.744 INFO:teuthology.orchestra.run.smithi022.stdout: "progress": "15/23 daemons upgraded", 2024-03-21T12:47:51.744 INFO:teuthology.orchestra.run.smithi022.stdout: "message": "Currently upgrading mds daemons", 2024-03-21T12:47:51.744 INFO:teuthology.orchestra.run.smithi022.stdout: "is_paused": false 2024-03-21T12:47:51.744 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-03-21T12:47:52.096 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:51 smithi022 ceph-mon[67047]: from='client.34500 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:47:52.096 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:51 smithi022 ceph-mon[67047]: pgmap v341: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 7.3 KiB/s wr, 14 op/s 2024-03-21T12:47:52.096 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:51 smithi022 ceph-mon[67047]: from='client.34504 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:47:52.097 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:51 smithi022 ceph-mon[67047]: from='client.? 172.21.15.22:0/3222437539' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:47:52.232 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:51 smithi189 ceph-mon[53160]: from='client.34500 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:47:52.232 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:51 smithi189 ceph-mon[53160]: pgmap v341: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 7.3 KiB/s wr, 14 op/s 2024-03-21T12:47:52.232 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:51 smithi189 ceph-mon[53160]: from='client.34504 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:47:52.233 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:51 smithi189 ceph-mon[53160]: from='client.? 172.21.15.22:0/3222437539' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:47:52.246 INFO:teuthology.orchestra.run.smithi022.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-03-21T12:47:52.246 INFO:teuthology.orchestra.run.smithi022.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-21T12:47:52.246 INFO:teuthology.orchestra.run.smithi022.stdout: daemon grafana.smithi022 on smithi022 is in error state 2024-03-21T12:47:53.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:52 smithi189 ceph-mon[53160]: from='client.34512 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:47:53.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:52 smithi189 ceph-mon[53160]: from='client.? 172.21.15.22:0/2227983191' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-21T12:47:53.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:52 smithi022 ceph-mon[67047]: from='client.34512 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:47:53.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:52 smithi022 ceph-mon[67047]: from='client.? 172.21.15.22:0/2227983191' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-21T12:47:54.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:54 smithi022 ceph-mon[67047]: pgmap v342: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 7.4 KiB/s wr, 12 op/s 2024-03-21T12:47:54.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:54 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:47:54.345 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:54 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:47:54.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:54 smithi189 ceph-mon[53160]: pgmap v342: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 7.4 KiB/s wr, 12 op/s 2024-03-21T12:47:54.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:54 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:47:54.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:54 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:47:55.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:55 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:47:55.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:55 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:47:55.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:55 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:47:55.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:55 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:47:56.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:56 smithi189 ceph-mon[53160]: pgmap v343: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 8.5 MiB/s rd, 7.3 KiB/s wr, 8 op/s 2024-03-21T12:47:56.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:56 smithi022 ceph-mon[67047]: pgmap v343: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 8.5 MiB/s rd, 7.3 KiB/s wr, 8 op/s 2024-03-21T12:47:58.491 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:58 smithi022 ceph-mon[67047]: pgmap v344: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.0 MiB/s rd, 7.3 KiB/s wr, 6 op/s 2024-03-21T12:47:58.491 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:58 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:47:58.491 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:58 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:47:58.491 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:58 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:47:58.491 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:58 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:47:58.491 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:58 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:47:58.491 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:58 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:47:58.491 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:58 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:47:58.491 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:58 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:47:58.491 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:58 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:47:58.491 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:58 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:47:58.491 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:58 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:47:58.491 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:58 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:47:58.491 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:58 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:47:58.492 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:58 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi022.bcnxho"}]: dispatch 2024-03-21T12:47:58.492 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:58 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi022.bcnxho"}]': finished 2024-03-21T12:47:58.492 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:58 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi189.lluvko"}]: dispatch 2024-03-21T12:47:58.492 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:58 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi189.lluvko"}]': finished 2024-03-21T12:47:58.492 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:58 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:47:58.492 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:58 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:47:58.492 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:58 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:47:58.492 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:58 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:47:58.492 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:58 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:47:58.492 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:58 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:47:58.492 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:58 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:47:58.492 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:58 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:47:58.492 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:58 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:47:58.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:58 smithi189 ceph-mon[53160]: pgmap v344: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.0 MiB/s rd, 7.3 KiB/s wr, 6 op/s 2024-03-21T12:47:58.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:58 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:47:58.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:58 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:47:58.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:58 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:47:58.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:58 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:47:58.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:58 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:47:58.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:58 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:47:58.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:58 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:47:58.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:58 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:47:58.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:58 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:47:58.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:58 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:47:58.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:58 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:47:58.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:58 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:47:58.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:58 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:47:58.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:58 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi022.bcnxho"}]: dispatch 2024-03-21T12:47:58.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:58 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi022.bcnxho"}]': finished 2024-03-21T12:47:58.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:58 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi189.lluvko"}]: dispatch 2024-03-21T12:47:58.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:58 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi189.lluvko"}]': finished 2024-03-21T12:47:58.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:58 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:47:58.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:58 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:47:58.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:58 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:47:58.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:58 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:47:58.538 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:58 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:47:58.538 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:58 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:47:58.538 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:58 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:47:58.538 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:58 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:47:58.538 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:58 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:47:59.508 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:59 smithi022 ceph-mon[67047]: Detected new or changed devices on smithi189 2024-03-21T12:47:59.508 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:59 smithi022 ceph-mon[67047]: pgmap v345: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4.0 KiB/s rd, 8.2 KiB/s wr, 7 op/s 2024-03-21T12:47:59.508 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:59 smithi022 ceph-mon[67047]: pgmap v346: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.9 KiB/s wr, 0 op/s 2024-03-21T12:47:59.508 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:59 smithi022 ceph-mon[67047]: Upgrade: Setting container_image for all mds 2024-03-21T12:47:59.508 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:59 smithi022 ceph-mon[67047]: Upgrade: Setting container_image for all rgw 2024-03-21T12:47:59.508 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:59 smithi022 ceph-mon[67047]: Upgrade: Setting container_image for all rbd-mirror 2024-03-21T12:47:59.508 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:59 smithi022 ceph-mon[67047]: Upgrade: Setting container_image for all ceph-exporter 2024-03-21T12:47:59.508 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:47:59 smithi022 ceph-mon[67047]: Upgrade: Setting container_image for all iscsi 2024-03-21T12:47:59.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:59 smithi189 ceph-mon[53160]: Detected new or changed devices on smithi189 2024-03-21T12:47:59.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:59 smithi189 ceph-mon[53160]: pgmap v345: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4.0 KiB/s rd, 8.2 KiB/s wr, 7 op/s 2024-03-21T12:47:59.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:59 smithi189 ceph-mon[53160]: pgmap v346: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.9 KiB/s wr, 0 op/s 2024-03-21T12:47:59.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:59 smithi189 ceph-mon[53160]: Upgrade: Setting container_image for all mds 2024-03-21T12:47:59.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:59 smithi189 ceph-mon[53160]: Upgrade: Setting container_image for all rgw 2024-03-21T12:47:59.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:59 smithi189 ceph-mon[53160]: Upgrade: Setting container_image for all rbd-mirror 2024-03-21T12:47:59.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:59 smithi189 ceph-mon[53160]: Upgrade: Setting container_image for all ceph-exporter 2024-03-21T12:47:59.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:47:59 smithi189 ceph-mon[53160]: Upgrade: Setting container_image for all iscsi 2024-03-21T12:48:00.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:00 smithi022 ceph-mon[67047]: Upgrade: Updating nfs.foo.0.0.smithi022.ynhbbq (1/2) 2024-03-21T12:48:00.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:00 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:00.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:00 smithi022 ceph-mon[67047]: Creating key for client.nfs.foo.0.0.smithi022.ynhbbq 2024-03-21T12:48:00.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:00 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi022.ynhbbq", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-03-21T12:48:00.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:00 smithi022 ceph-mon[67047]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-03-21T12:48:00.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:00 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-03-21T12:48:00.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:00 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-03-21T12:48:00.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:00 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:48:00.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:00 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-03-21T12:48:00.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:00 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-03-21T12:48:00.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:00 smithi022 ceph-mon[67047]: Rados config object exists: conf-nfs.foo 2024-03-21T12:48:00.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:00 smithi022 ceph-mon[67047]: Creating key for client.nfs.foo.0.0.smithi022.ynhbbq-rgw 2024-03-21T12:48:00.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:00 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi022.ynhbbq-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-03-21T12:48:00.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:00 smithi022 ceph-mon[67047]: Bind address in nfs.foo.0.0.smithi022.ynhbbq's ganesha conf is defaulting to empty 2024-03-21T12:48:00.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:00 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:48:00.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:00 smithi022 ceph-mon[67047]: Deploying daemon nfs.foo.0.0.smithi022.ynhbbq on smithi022 2024-03-21T12:48:00.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:00 smithi022 ceph-mon[67047]: pgmap v347: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 228 B/s rd, 1.9 KiB/s wr, 0 op/s 2024-03-21T12:48:01.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:00 smithi189 ceph-mon[53160]: Upgrade: Updating nfs.foo.0.0.smithi022.ynhbbq (1/2) 2024-03-21T12:48:01.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:00 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:01.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:00 smithi189 ceph-mon[53160]: Creating key for client.nfs.foo.0.0.smithi022.ynhbbq 2024-03-21T12:48:01.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:00 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi022.ynhbbq", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-03-21T12:48:01.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:00 smithi189 ceph-mon[53160]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-03-21T12:48:01.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:00 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-03-21T12:48:01.037 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:00 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-03-21T12:48:01.037 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:00 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:48:01.037 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:00 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-03-21T12:48:01.037 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:00 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-03-21T12:48:01.037 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:00 smithi189 ceph-mon[53160]: Rados config object exists: conf-nfs.foo 2024-03-21T12:48:01.037 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:00 smithi189 ceph-mon[53160]: Creating key for client.nfs.foo.0.0.smithi022.ynhbbq-rgw 2024-03-21T12:48:01.037 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:00 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi022.ynhbbq-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-03-21T12:48:01.037 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:00 smithi189 ceph-mon[53160]: Bind address in nfs.foo.0.0.smithi022.ynhbbq's ganesha conf is defaulting to empty 2024-03-21T12:48:01.037 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:00 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:48:01.037 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:00 smithi189 ceph-mon[53160]: Deploying daemon nfs.foo.0.0.smithi022.ynhbbq on smithi022 2024-03-21T12:48:01.037 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:00 smithi189 ceph-mon[53160]: pgmap v347: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 228 B/s rd, 1.9 KiB/s wr, 0 op/s 2024-03-21T12:48:02.831 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:02 smithi022 ceph-mon[67047]: pgmap v348: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 228 B/s rd, 228 B/s wr, 0 op/s 2024-03-21T12:48:03.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:02 smithi189 ceph-mon[53160]: pgmap v348: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 228 B/s rd, 228 B/s wr, 0 op/s 2024-03-21T12:48:04.035 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:03 smithi189 ceph-mon[53160]: from='mds.? [v2:172.21.15.22:6834/1911267561,v1:172.21.15.22:6835/1911267561]' entity='mds.foofs.smithi022.bcnxho' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.22:0/3037192481"}]: dispatch 2024-03-21T12:48:04.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:03 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:04.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:03 smithi022 ceph-mon[67047]: from='mds.? [v2:172.21.15.22:6834/1911267561,v1:172.21.15.22:6835/1911267561]' entity='mds.foofs.smithi022.bcnxho' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.22:0/3037192481"}]: dispatch 2024-03-21T12:48:04.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:03 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:05.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:04 smithi189 ceph-mon[53160]: Evicting (and blocklisting) client session 14546 (172.21.15.22:0/3037192481) 2024-03-21T12:48:05.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:04 smithi189 ceph-mon[53160]: pgmap v349: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.4 KiB/s rd, 570 B/s wr, 1 op/s 2024-03-21T12:48:05.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:04 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:05.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:04 smithi189 ceph-mon[53160]: from='mds.? [v2:172.21.15.22:6834/1911267561,v1:172.21.15.22:6835/1911267561]' entity='mds.foofs.smithi022.bcnxho' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.22:0/3037192481"}]': finished 2024-03-21T12:48:05.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:04 smithi189 ceph-mon[53160]: osdmap e163: 8 total, 8 up, 8 in 2024-03-21T12:48:05.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:04 smithi022 ceph-mon[67047]: Evicting (and blocklisting) client session 14546 (172.21.15.22:0/3037192481) 2024-03-21T12:48:05.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:04 smithi022 ceph-mon[67047]: pgmap v349: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.4 KiB/s rd, 570 B/s wr, 1 op/s 2024-03-21T12:48:05.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:04 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:05.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:04 smithi022 ceph-mon[67047]: from='mds.? [v2:172.21.15.22:6834/1911267561,v1:172.21.15.22:6835/1911267561]' entity='mds.foofs.smithi022.bcnxho' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.22:0/3037192481"}]': finished 2024-03-21T12:48:05.345 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:04 smithi022 ceph-mon[67047]: osdmap e163: 8 total, 8 up, 8 in 2024-03-21T12:48:06.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:06 smithi189 ceph-mon[53160]: Upgrade: Updating nfs.foo.1.0.smithi189.yzsqml (2/2) 2024-03-21T12:48:06.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:06 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:06.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:06 smithi189 ceph-mon[53160]: Creating key for client.nfs.foo.1.0.smithi189.yzsqml 2024-03-21T12:48:06.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:06 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi189.yzsqml", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-03-21T12:48:06.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:06 smithi189 ceph-mon[53160]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-03-21T12:48:06.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:06 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-03-21T12:48:06.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:06 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-03-21T12:48:06.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:06 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:48:06.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:06 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-03-21T12:48:06.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:06 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-03-21T12:48:06.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:06 smithi189 ceph-mon[53160]: Rados config object exists: conf-nfs.foo 2024-03-21T12:48:06.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:06 smithi189 ceph-mon[53160]: Creating key for client.nfs.foo.1.0.smithi189.yzsqml-rgw 2024-03-21T12:48:06.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:06 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi189.yzsqml-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-03-21T12:48:06.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:06 smithi189 ceph-mon[53160]: Bind address in nfs.foo.1.0.smithi189.yzsqml's ganesha conf is defaulting to empty 2024-03-21T12:48:06.787 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:06 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:48:06.787 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:06 smithi189 ceph-mon[53160]: Deploying daemon nfs.foo.1.0.smithi189.yzsqml on smithi189 2024-03-21T12:48:06.787 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:06 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:48:06.787 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:06 smithi189 ceph-mon[53160]: pgmap v351: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.6 KiB/s rd, 511 B/s wr, 2 op/s 2024-03-21T12:48:06.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:06 smithi022 ceph-mon[67047]: Upgrade: Updating nfs.foo.1.0.smithi189.yzsqml (2/2) 2024-03-21T12:48:06.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:06 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:06.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:06 smithi022 ceph-mon[67047]: Creating key for client.nfs.foo.1.0.smithi189.yzsqml 2024-03-21T12:48:06.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:06 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi189.yzsqml", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-03-21T12:48:06.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:06 smithi022 ceph-mon[67047]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-03-21T12:48:06.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:06 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-03-21T12:48:06.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:06 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-03-21T12:48:06.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:06 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:48:06.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:06 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-03-21T12:48:06.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:06 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-03-21T12:48:06.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:06 smithi022 ceph-mon[67047]: Rados config object exists: conf-nfs.foo 2024-03-21T12:48:06.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:06 smithi022 ceph-mon[67047]: Creating key for client.nfs.foo.1.0.smithi189.yzsqml-rgw 2024-03-21T12:48:06.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:06 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi189.yzsqml-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-03-21T12:48:06.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:06 smithi022 ceph-mon[67047]: Bind address in nfs.foo.1.0.smithi189.yzsqml's ganesha conf is defaulting to empty 2024-03-21T12:48:06.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:06 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:48:06.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:06 smithi022 ceph-mon[67047]: Deploying daemon nfs.foo.1.0.smithi189.yzsqml on smithi189 2024-03-21T12:48:06.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:06 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:48:06.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:06 smithi022 ceph-mon[67047]: pgmap v351: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.6 KiB/s rd, 511 B/s wr, 2 op/s 2024-03-21T12:48:09.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:08 smithi189 ceph-mon[53160]: pgmap v352: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.5 KiB/s rd, 921 B/s wr, 2 op/s 2024-03-21T12:48:09.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:08 smithi022 ceph-mon[67047]: pgmap v352: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.5 KiB/s rd, 921 B/s wr, 2 op/s 2024-03-21T12:48:10.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:10 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:10.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:10 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:10.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:10 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:48:10.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:10 smithi189 ceph-mon[53160]: pgmap v353: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 3.1 KiB/s rd, 1.3 KiB/s wr, 4 op/s 2024-03-21T12:48:10.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:10 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:10.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:10 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:10.595 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:10 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:48:10.595 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:10 smithi022 ceph-mon[67047]: pgmap v353: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 3.1 KiB/s rd, 1.3 KiB/s wr, 4 op/s 2024-03-21T12:48:13.054 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:12 smithi022 ceph-mon[67047]: pgmap v354: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 3.1 KiB/s rd, 1.4 KiB/s wr, 4 op/s 2024-03-21T12:48:13.054 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:12 smithi022 ceph-mon[67047]: from='mds.? [v2:172.21.15.22:6834/1911267561,v1:172.21.15.22:6835/1911267561]' entity='mds.foofs.smithi022.bcnxho' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.189:0/4097293788"}]: dispatch 2024-03-21T12:48:13.054 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:12 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:13.054 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:12 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:13.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:12 smithi189 ceph-mon[53160]: pgmap v354: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 3.1 KiB/s rd, 1.4 KiB/s wr, 4 op/s 2024-03-21T12:48:13.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:12 smithi189 ceph-mon[53160]: from='mds.? [v2:172.21.15.22:6834/1911267561,v1:172.21.15.22:6835/1911267561]' entity='mds.foofs.smithi022.bcnxho' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.189:0/4097293788"}]: dispatch 2024-03-21T12:48:13.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:12 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:13.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:12 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:14.062 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:13 smithi022 ceph-mon[67047]: Evicting (and blocklisting) client session 24359 (172.21.15.189:0/4097293788) 2024-03-21T12:48:14.062 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:13 smithi022 ceph-mon[67047]: from='mds.? [v2:172.21.15.22:6834/1911267561,v1:172.21.15.22:6835/1911267561]' entity='mds.foofs.smithi022.bcnxho' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.189:0/4097293788"}]': finished 2024-03-21T12:48:14.062 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:13 smithi022 ceph-mon[67047]: osdmap e164: 8 total, 8 up, 8 in 2024-03-21T12:48:14.062 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:13 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:14.062 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:13 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:14.185 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:13 smithi189 ceph-mon[53160]: Evicting (and blocklisting) client session 24359 (172.21.15.189:0/4097293788) 2024-03-21T12:48:14.185 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:13 smithi189 ceph-mon[53160]: from='mds.? [v2:172.21.15.22:6834/1911267561,v1:172.21.15.22:6835/1911267561]' entity='mds.foofs.smithi022.bcnxho' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.189:0/4097293788"}]': finished 2024-03-21T12:48:14.185 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:13 smithi189 ceph-mon[53160]: osdmap e164: 8 total, 8 up, 8 in 2024-03-21T12:48:14.185 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:13 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:14.185 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:13 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:15.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:14 smithi022 ceph-mon[67047]: pgmap v356: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4.4 KiB/s rd, 1.7 KiB/s wr, 6 op/s 2024-03-21T12:48:15.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:14 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:15.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:14 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:15.218 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:14 smithi189 ceph-mon[53160]: pgmap v356: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4.4 KiB/s rd, 1.7 KiB/s wr, 6 op/s 2024-03-21T12:48:15.218 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:14 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:15.218 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:14 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:16.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:16 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:16.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:16 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:16.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:16 smithi189 ceph-mon[53160]: pgmap v357: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4.3 KiB/s rd, 1.6 KiB/s wr, 6 op/s 2024-03-21T12:48:16.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:16 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:16.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:16 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:16.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:16 smithi022 ceph-mon[67047]: pgmap v357: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4.3 KiB/s rd, 1.6 KiB/s wr, 6 op/s 2024-03-21T12:48:18.459 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:18 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:18.459 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:18 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:18.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:18 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:18.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:18 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:19.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:19 smithi022 ceph-mon[67047]: pgmap v358: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4.1 KiB/s rd, 1.5 KiB/s wr, 6 op/s 2024-03-21T12:48:19.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:19 smithi189 ceph-mon[53160]: pgmap v358: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4.1 KiB/s rd, 1.5 KiB/s wr, 6 op/s 2024-03-21T12:48:20.466 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:20 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:20.467 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:20 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:20.467 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:20 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:48:20.467 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:20 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:48:20.467 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:20 smithi022 ceph-mon[67047]: pgmap v359: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.4 KiB/s rd, 1.1 KiB/s wr, 4 op/s 2024-03-21T12:48:20.467 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:20 smithi022 ceph-mon[67047]: pgmap v360: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 3.0 KiB/s rd, 1.3 KiB/s wr, 4 op/s 2024-03-21T12:48:20.467 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:20 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:20.467 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:20 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:20.467 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:20 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:48:20.467 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:20 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:48:20.467 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:20 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:48:20.467 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:20 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:48:20.467 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:20 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:48:20.467 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:20 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:48:20.467 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:20 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:48:20.467 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:20 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:48:20.468 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:20 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:48:20.468 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:20 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:48:20.468 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:20 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:48:20.468 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:20 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:48:20.468 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:20 smithi022 ceph-mon[67047]: Upgrade: Setting container_image for all nfs 2024-03-21T12:48:20.468 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:20 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:20.468 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:20 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi022.ynhbbq"}]: dispatch 2024-03-21T12:48:20.468 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:20 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi022.ynhbbq"}]': finished 2024-03-21T12:48:20.468 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:20 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi189.yzsqml"}]: dispatch 2024-03-21T12:48:20.468 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:20 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi189.yzsqml"}]': finished 2024-03-21T12:48:20.468 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:20 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:48:20.468 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:20 smithi022 ceph-mon[67047]: Upgrade: Setting container_image for all nvmeof 2024-03-21T12:48:20.468 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:20 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:20.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:20 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:20.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:20 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:20.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:20 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:48:20.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:20 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:48:20.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:20 smithi189 ceph-mon[53160]: pgmap v359: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.4 KiB/s rd, 1.1 KiB/s wr, 4 op/s 2024-03-21T12:48:20.787 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:20 smithi189 ceph-mon[53160]: pgmap v360: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 3.0 KiB/s rd, 1.3 KiB/s wr, 4 op/s 2024-03-21T12:48:20.787 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:20 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:20.787 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:20 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:20.787 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:20 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:48:20.787 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:20 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:48:20.787 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:20 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:48:20.787 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:20 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:48:20.787 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:20 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:48:20.787 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:20 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:48:20.787 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:20 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:48:20.787 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:20 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:48:20.787 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:20 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:48:20.787 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:20 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:48:20.787 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:20 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:48:20.787 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:20 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:48:20.787 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:20 smithi189 ceph-mon[53160]: Upgrade: Setting container_image for all nfs 2024-03-21T12:48:20.788 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:20 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:20.788 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:20 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi022.ynhbbq"}]: dispatch 2024-03-21T12:48:20.788 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:20 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi022.ynhbbq"}]': finished 2024-03-21T12:48:20.788 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:20 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi189.yzsqml"}]: dispatch 2024-03-21T12:48:20.788 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:20 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi189.yzsqml"}]': finished 2024-03-21T12:48:20.788 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:20 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:48:20.788 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:20 smithi189 ceph-mon[53160]: Upgrade: Setting container_image for all nvmeof 2024-03-21T12:48:20.788 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:20 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:21.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:21 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:48:21.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:21 smithi022 ceph-mon[67047]: Upgrade: Updating node-exporter.smithi022 (1/2) 2024-03-21T12:48:21.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:21 smithi022 ceph-mon[67047]: Deploying daemon node-exporter.smithi022 on smithi022 2024-03-21T12:48:21.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:21 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:48:21.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:21 smithi189 ceph-mon[53160]: Upgrade: Updating node-exporter.smithi022 (1/2) 2024-03-21T12:48:21.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:21 smithi189 ceph-mon[53160]: Deploying daemon node-exporter.smithi022 on smithi022 2024-03-21T12:48:22.699 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:22 smithi022 ceph-mon[67047]: pgmap v361: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.5 KiB/s rd, 1.2 KiB/s wr, 4 op/s 2024-03-21T12:48:22.742 INFO:teuthology.orchestra.run.smithi022.stdout:true 2024-03-21T12:48:22.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:22 smithi189 ceph-mon[53160]: pgmap v361: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.5 KiB/s rd, 1.2 KiB/s wr, 4 op/s 2024-03-21T12:48:23.499 INFO:teuthology.orchestra.run.smithi022.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-21T12:48:23.499 INFO:teuthology.orchestra.run.smithi022.stdout:alertmanager.smithi022 smithi022 *:9093,9094 running (10m) 9s ago 19m 24.5M - 0.23.0 ba2b418f427c 14eb2a3ca318 2024-03-21T12:48:23.499 INFO:teuthology.orchestra.run.smithi022.stdout:crash.smithi022 smithi022 running (7m) 9s ago 19m 6660k - 19.0.0-2267-g960fbbe4 19e12293512f 356b2ddb609a 2024-03-21T12:48:23.500 INFO:teuthology.orchestra.run.smithi022.stdout:crash.smithi189 smithi189 running (6m) 11s ago 19m 6660k - 19.0.0-2267-g960fbbe4 19e12293512f 90ee837f0500 2024-03-21T12:48:23.500 INFO:teuthology.orchestra.run.smithi022.stdout:grafana.smithi022 smithi022 *:3000 error 9s ago 19m - - 2024-03-21T12:48:23.500 INFO:teuthology.orchestra.run.smithi022.stdout:mds.foofs.smithi022.bcnxho smithi022 running (57s) 9s ago 13m 33.7M - 19.0.0-2267-g960fbbe4 19e12293512f 6db0e27a3822 2024-03-21T12:48:23.500 INFO:teuthology.orchestra.run.smithi022.stdout:mds.foofs.smithi189.lluvko smithi189 running (34s) 11s ago 13m 12.5M - 19.0.0-2267-g960fbbe4 19e12293512f ee1f46135bb3 2024-03-21T12:48:23.500 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.smithi022.fektzs smithi022 *:8443,9283,8765 running (9m) 9s ago 20m 511M - 19.0.0-2267-g960fbbe4 19e12293512f d57cebd306b3 2024-03-21T12:48:23.500 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.smithi189.cnxzvu smithi189 *:8443,9283,8765 running (8m) 11s ago 19m 453M - 19.0.0-2267-g960fbbe4 19e12293512f 41663955c56f 2024-03-21T12:48:23.500 INFO:teuthology.orchestra.run.smithi022.stdout:mon.smithi022 smithi022 running (8m) 9s ago 21m 62.3M 2048M 19.0.0-2267-g960fbbe4 19e12293512f c620c9071b2c 2024-03-21T12:48:23.500 INFO:teuthology.orchestra.run.smithi022.stdout:mon.smithi189 smithi189 running (7m) 11s ago 18m 47.9M 2048M 19.0.0-2267-g960fbbe4 19e12293512f b353af3d8f3b 2024-03-21T12:48:23.500 INFO:teuthology.orchestra.run.smithi022.stdout:nfs.foo.0.0.smithi022.ynhbbq smithi022 *:2049 running (20s) 9s ago 13m 71.7M - 5.7 19e12293512f 825c79cba449 2024-03-21T12:48:23.500 INFO:teuthology.orchestra.run.smithi022.stdout:nfs.foo.1.0.smithi189.yzsqml smithi189 *:2049 running (14s) 11s ago 13m 18.4M - 5.7 19e12293512f f6ea207ea727 2024-03-21T12:48:23.500 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.smithi022 smithi022 *:9100 running (10m) 9s ago 19m 15.8M - 1.3.1 1dbe0e931976 33287f803f4f 2024-03-21T12:48:23.500 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.smithi189 smithi189 *:9100 running (10m) 11s ago 18m 20.2M - 1.3.1 1dbe0e931976 b51d872e438a 2024-03-21T12:48:23.500 INFO:teuthology.orchestra.run.smithi022.stdout:osd.0 smithi022 running (6m) 9s ago 18m 406M 1937M 19.0.0-2267-g960fbbe4 19e12293512f 23a85645eeea 2024-03-21T12:48:23.500 INFO:teuthology.orchestra.run.smithi022.stdout:osd.1 smithi022 running (5m) 9s ago 17m 383M 1937M 19.0.0-2267-g960fbbe4 19e12293512f 0f90e70bbf0a 2024-03-21T12:48:23.500 INFO:teuthology.orchestra.run.smithi022.stdout:osd.2 smithi022 running (5m) 9s ago 17m 356M 1937M 19.0.0-2267-g960fbbe4 19e12293512f 78a229f593f8 2024-03-21T12:48:23.500 INFO:teuthology.orchestra.run.smithi022.stdout:osd.3 smithi022 running (4m) 9s ago 16m 443M 1937M 19.0.0-2267-g960fbbe4 19e12293512f 61a63b67d9af 2024-03-21T12:48:23.500 INFO:teuthology.orchestra.run.smithi022.stdout:osd.4 smithi189 running (3m) 11s ago 16m 369M 2704M 19.0.0-2267-g960fbbe4 19e12293512f 382755cee998 2024-03-21T12:48:23.501 INFO:teuthology.orchestra.run.smithi022.stdout:osd.5 smithi189 running (2m) 11s ago 16m 332M 2704M 19.0.0-2267-g960fbbe4 19e12293512f d2c15ba54b4f 2024-03-21T12:48:23.501 INFO:teuthology.orchestra.run.smithi022.stdout:osd.6 smithi189 running (2m) 11s ago 15m 314M 2704M 19.0.0-2267-g960fbbe4 19e12293512f 291089fb3596 2024-03-21T12:48:23.501 INFO:teuthology.orchestra.run.smithi022.stdout:osd.7 smithi189 running (86s) 11s ago 15m 311M 2704M 19.0.0-2267-g960fbbe4 19e12293512f 247b82310a8a 2024-03-21T12:48:23.501 INFO:teuthology.orchestra.run.smithi022.stdout:prometheus.smithi022 smithi022 *:9095 running (8m) 9s ago 18m 77.1M - 2.33.4 514e6a882f6e 8171bbe41d2c 2024-03-21T12:48:23.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:23 smithi022 ceph-mon[67047]: from='client.34562 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:48:23.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:23 smithi022 ceph-mon[67047]: from='client.34566 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:48:23.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:23 smithi189 ceph-mon[53160]: from='client.34562 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:48:23.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:23 smithi189 ceph-mon[53160]: from='client.34566 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:48:24.011 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-03-21T12:48:24.011 INFO:teuthology.orchestra.run.smithi022.stdout: "mon": { 2024-03-21T12:48:24.011 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 2 2024-03-21T12:48:24.011 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:48:24.012 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr": { 2024-03-21T12:48:24.012 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 2 2024-03-21T12:48:24.012 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:48:24.012 INFO:teuthology.orchestra.run.smithi022.stdout: "osd": { 2024-03-21T12:48:24.012 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 8 2024-03-21T12:48:24.012 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:48:24.012 INFO:teuthology.orchestra.run.smithi022.stdout: "mds": { 2024-03-21T12:48:24.012 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 2 2024-03-21T12:48:24.012 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:48:24.012 INFO:teuthology.orchestra.run.smithi022.stdout: "overall": { 2024-03-21T12:48:24.012 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 14 2024-03-21T12:48:24.012 INFO:teuthology.orchestra.run.smithi022.stdout: } 2024-03-21T12:48:24.012 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-03-21T12:48:24.400 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-03-21T12:48:24.400 INFO:teuthology.orchestra.run.smithi022.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93", 2024-03-21T12:48:24.400 INFO:teuthology.orchestra.run.smithi022.stdout: "in_progress": true, 2024-03-21T12:48:24.400 INFO:teuthology.orchestra.run.smithi022.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-21T12:48:24.400 INFO:teuthology.orchestra.run.smithi022.stdout: "services_complete": [ 2024-03-21T12:48:24.400 INFO:teuthology.orchestra.run.smithi022.stdout: "mon", 2024-03-21T12:48:24.400 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr", 2024-03-21T12:48:24.401 INFO:teuthology.orchestra.run.smithi022.stdout: "osd", 2024-03-21T12:48:24.401 INFO:teuthology.orchestra.run.smithi022.stdout: "crash", 2024-03-21T12:48:24.401 INFO:teuthology.orchestra.run.smithi022.stdout: "mds", 2024-03-21T12:48:24.401 INFO:teuthology.orchestra.run.smithi022.stdout: "nfs" 2024-03-21T12:48:24.401 INFO:teuthology.orchestra.run.smithi022.stdout: ], 2024-03-21T12:48:24.401 INFO:teuthology.orchestra.run.smithi022.stdout: "progress": "18/23 daemons upgraded", 2024-03-21T12:48:24.401 INFO:teuthology.orchestra.run.smithi022.stdout: "message": "Currently upgrading node-exporter daemons", 2024-03-21T12:48:24.401 INFO:teuthology.orchestra.run.smithi022.stdout: "is_paused": false 2024-03-21T12:48:24.401 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-03-21T12:48:24.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:24 smithi022 ceph-mon[67047]: pgmap v362: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 211 B/s rd, 422 B/s wr, 0 op/s 2024-03-21T12:48:24.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:24 smithi022 ceph-mon[67047]: from='client.34570 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:48:24.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:24 smithi022 ceph-mon[67047]: from='client.? 172.21.15.22:0/3591045197' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:48:24.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:24 smithi022 ceph-mon[67047]: from='client.34578 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:48:24.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:24 smithi189 ceph-mon[53160]: pgmap v362: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 211 B/s rd, 422 B/s wr, 0 op/s 2024-03-21T12:48:24.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:24 smithi189 ceph-mon[53160]: from='client.34570 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:48:24.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:24 smithi189 ceph-mon[53160]: from='client.? 172.21.15.22:0/3591045197' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:48:24.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:24 smithi189 ceph-mon[53160]: from='client.34578 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:48:24.922 INFO:teuthology.orchestra.run.smithi022.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-03-21T12:48:24.922 INFO:teuthology.orchestra.run.smithi022.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-21T12:48:24.922 INFO:teuthology.orchestra.run.smithi022.stdout: daemon grafana.smithi022 on smithi022 is in error state 2024-03-21T12:48:25.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:25 smithi189 ceph-mon[53160]: from='client.? 172.21.15.22:0/3952327398' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-21T12:48:25.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:25 smithi022 ceph-mon[67047]: from='client.? 172.21.15.22:0/3952327398' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-21T12:48:26.715 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:26 smithi022 ceph-mon[67047]: pgmap v363: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 211 B/s rd, 422 B/s wr, 0 op/s 2024-03-21T12:48:26.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:26 smithi189 ceph-mon[53160]: pgmap v363: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 211 B/s rd, 422 B/s wr, 0 op/s 2024-03-21T12:48:28.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:27 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:28.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:27 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:28.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:27 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:28.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:27 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:29.260 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:28 smithi189 ceph-mon[53160]: pgmap v364: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 211 B/s rd, 0 op/s 2024-03-21T12:48:29.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:28 smithi022 ceph-mon[67047]: pgmap v364: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 211 B/s rd, 0 op/s 2024-03-21T12:48:29.983 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:29 smithi189 ceph-mon[53160]: Upgrade: Updating node-exporter.smithi189 (2/2) 2024-03-21T12:48:29.983 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:29 smithi189 ceph-mon[53160]: Deploying daemon node-exporter.smithi189 on smithi189 2024-03-21T12:48:30.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:29 smithi022 ceph-mon[67047]: Upgrade: Updating node-exporter.smithi189 (2/2) 2024-03-21T12:48:30.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:29 smithi022 ceph-mon[67047]: Deploying daemon node-exporter.smithi189 on smithi189 2024-03-21T12:48:31.222 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:30 smithi189 ceph-mon[53160]: pgmap v365: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-03-21T12:48:31.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:30 smithi022 ceph-mon[67047]: pgmap v365: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-03-21T12:48:33.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:32 smithi189 ceph-mon[53160]: pgmap v366: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:48:33.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:32 smithi022 ceph-mon[67047]: pgmap v366: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:48:34.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:34 smithi189 ceph-mon[53160]: pgmap v367: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-03-21T12:48:34.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:34 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:34.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:34 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:34.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:34 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:48:34.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:34 smithi022 ceph-mon[67047]: pgmap v367: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-03-21T12:48:34.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:34 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:34.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:34 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:34.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:34 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:48:37.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:36 smithi189 ceph-mon[53160]: pgmap v368: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:48:37.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:36 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:48:37.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:36 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:37.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:36 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:37.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:36 smithi022 ceph-mon[67047]: pgmap v368: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:48:37.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:36 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:48:37.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:36 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:37.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:36 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:39.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:38 smithi189 ceph-mon[53160]: pgmap v369: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:48:39.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:38 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:39.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:38 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:39.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:38 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:39.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:38 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:39.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:38 smithi022 ceph-mon[67047]: pgmap v369: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:48:39.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:38 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:39.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:38 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:39.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:38 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:39.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:38 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:41.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:40 smithi022 ceph-mon[67047]: pgmap v370: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:48:41.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:40 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:41.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:40 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:41.096 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:40 smithi189 ceph-mon[53160]: pgmap v370: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:48:41.096 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:40 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:41.096 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:40 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:42.444 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:42 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:42.445 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:42 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:42.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:42 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:42.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:42 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:43.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:43 smithi022 ceph-mon[67047]: pgmap v371: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:48:43.473 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:43 smithi189 ceph-mon[53160]: pgmap v371: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:48:44.532 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:44 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:44.532 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:44 smithi022 ceph-mon[67047]: pgmap v372: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-03-21T12:48:44.532 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:44 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:44.532 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:44 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:48:44.532 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:44 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:48:44.532 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:44 smithi022 ceph-mon[67047]: pgmap v373: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-03-21T12:48:44.532 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:44 smithi022 ceph-mon[67047]: pgmap v374: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-03-21T12:48:44.533 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:44 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:44.533 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:44 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:44.533 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:44 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:48:44.533 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:44 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:48:44.533 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:44 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:48:44.533 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:44 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:48:44.533 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:44 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:48:44.533 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:44 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:48:44.533 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:44 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:48:44.533 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:44 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:48:44.533 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:44 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:48:44.533 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:44 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:48:44.533 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:44 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:48:44.533 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:44 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:48:44.533 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:44 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:48:44.533 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:44 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:48:44.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:44 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:44.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:44 smithi189 ceph-mon[53160]: pgmap v372: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-03-21T12:48:44.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:44 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:44.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:44 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:48:44.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:44 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:48:44.787 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:44 smithi189 ceph-mon[53160]: pgmap v373: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-03-21T12:48:44.787 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:44 smithi189 ceph-mon[53160]: pgmap v374: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-03-21T12:48:44.787 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:44 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:44.787 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:44 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:44.787 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:44 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:48:44.787 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:44 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:48:44.787 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:44 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:48:44.787 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:44 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:48:44.787 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:44 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:48:44.787 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:44 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:48:44.787 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:44 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:48:44.787 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:44 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:48:44.787 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:44 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:48:44.787 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:44 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:48:44.787 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:44 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:48:44.788 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:44 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:48:44.788 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:44 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:48:44.788 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:44 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:48:45.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:45 smithi022 ceph-mon[67047]: Upgrade: Updating prometheus.smithi022 2024-03-21T12:48:45.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:45 smithi189 ceph-mon[53160]: Upgrade: Updating prometheus.smithi022 2024-03-21T12:48:46.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:46 smithi189 ceph-mon[53160]: pgmap v375: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-03-21T12:48:46.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:46 smithi189 ceph-mon[53160]: Deploying daemon prometheus.smithi022 on smithi022 2024-03-21T12:48:46.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:46 smithi022 ceph-mon[67047]: pgmap v375: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-03-21T12:48:46.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:46 smithi022 ceph-mon[67047]: Deploying daemon prometheus.smithi022 on smithi022 2024-03-21T12:48:48.702 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:48 smithi022 ceph-mon[67047]: pgmap v376: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-03-21T12:48:48.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:48 smithi189 ceph-mon[53160]: pgmap v376: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-03-21T12:48:50.710 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:50 smithi022 ceph-mon[67047]: pgmap v377: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-03-21T12:48:50.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:50 smithi189 ceph-mon[53160]: pgmap v377: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-03-21T12:48:51.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:51 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:48:51.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:51 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:51.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:51 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:51.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:51 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:48:51.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:51 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:48:51.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:51 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:51.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:51 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:51.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:51 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:48:52.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:52 smithi189 ceph-mon[53160]: pgmap v378: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-03-21T12:48:52.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:52 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:52.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:52 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:52.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:52 smithi022 ceph-mon[67047]: pgmap v378: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-03-21T12:48:52.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:52 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:52.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:52 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:54.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:54 smithi022 ceph-mon[67047]: pgmap v379: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-03-21T12:48:54.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:54 smithi189 ceph-mon[53160]: pgmap v379: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-03-21T12:48:55.296 INFO:teuthology.orchestra.run.smithi022.stdout:true 2024-03-21T12:48:55.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:55 smithi022 ceph-mon[67047]: from='client.34586 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:48:55.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:55 smithi189 ceph-mon[53160]: from='client.34586 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:48:56.045 INFO:teuthology.orchestra.run.smithi022.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-21T12:48:56.045 INFO:teuthology.orchestra.run.smithi022.stdout:alertmanager.smithi022 smithi022 *:9093,9094 running (11m) 17s ago 20m 24.1M - 0.23.0 ba2b418f427c 14eb2a3ca318 2024-03-21T12:48:56.045 INFO:teuthology.orchestra.run.smithi022.stdout:crash.smithi022 smithi022 running (7m) 17s ago 20m 6660k - 19.0.0-2267-g960fbbe4 19e12293512f 356b2ddb609a 2024-03-21T12:48:56.045 INFO:teuthology.orchestra.run.smithi022.stdout:crash.smithi189 smithi189 running (7m) 19s ago 19m 6660k - 19.0.0-2267-g960fbbe4 19e12293512f 90ee837f0500 2024-03-21T12:48:56.045 INFO:teuthology.orchestra.run.smithi022.stdout:grafana.smithi022 smithi022 *:3000 error 17s ago 20m - - 2024-03-21T12:48:56.045 INFO:teuthology.orchestra.run.smithi022.stdout:mds.foofs.smithi022.bcnxho smithi022 running (90s) 17s ago 14m 33.9M - 19.0.0-2267-g960fbbe4 19e12293512f 6db0e27a3822 2024-03-21T12:48:56.045 INFO:teuthology.orchestra.run.smithi022.stdout:mds.foofs.smithi189.lluvko smithi189 running (66s) 19s ago 14m 12.6M - 19.0.0-2267-g960fbbe4 19e12293512f ee1f46135bb3 2024-03-21T12:48:56.045 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.smithi022.fektzs smithi022 *:8443,9283,8765 running (10m) 17s ago 21m 512M - 19.0.0-2267-g960fbbe4 19e12293512f d57cebd306b3 2024-03-21T12:48:56.045 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.smithi189.cnxzvu smithi189 *:8443,9283,8765 running (9m) 19s ago 19m 453M - 19.0.0-2267-g960fbbe4 19e12293512f 41663955c56f 2024-03-21T12:48:56.045 INFO:teuthology.orchestra.run.smithi022.stdout:mon.smithi022 smithi022 running (9m) 17s ago 21m 65.5M 2048M 19.0.0-2267-g960fbbe4 19e12293512f c620c9071b2c 2024-03-21T12:48:56.045 INFO:teuthology.orchestra.run.smithi022.stdout:mon.smithi189 smithi189 running (8m) 19s ago 19m 51.3M 2048M 19.0.0-2267-g960fbbe4 19e12293512f b353af3d8f3b 2024-03-21T12:48:56.045 INFO:teuthology.orchestra.run.smithi022.stdout:nfs.foo.0.0.smithi022.ynhbbq smithi022 *:2049 running (52s) 17s ago 14m 71.7M - 5.7 19e12293512f 825c79cba449 2024-03-21T12:48:56.045 INFO:teuthology.orchestra.run.smithi022.stdout:nfs.foo.1.0.smithi189.yzsqml smithi189 *:2049 running (47s) 19s ago 14m 68.7M - 5.7 19e12293512f f6ea207ea727 2024-03-21T12:48:56.046 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.smithi022 smithi022 *:9100 running (29s) 17s ago 19m 5440k - 1.5.0 0da6a335fe13 a05caa6c4bcb 2024-03-21T12:48:56.046 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.smithi189 smithi189 *:9100 running (23s) 19s ago 19m 5435k - 1.5.0 0da6a335fe13 541a9c8dca57 2024-03-21T12:48:56.046 INFO:teuthology.orchestra.run.smithi022.stdout:osd.0 smithi022 running (6m) 17s ago 18m 406M 1937M 19.0.0-2267-g960fbbe4 19e12293512f 23a85645eeea 2024-03-21T12:48:56.046 INFO:teuthology.orchestra.run.smithi022.stdout:osd.1 smithi022 running (6m) 17s ago 18m 383M 1937M 19.0.0-2267-g960fbbe4 19e12293512f 0f90e70bbf0a 2024-03-21T12:48:56.046 INFO:teuthology.orchestra.run.smithi022.stdout:osd.2 smithi022 running (5m) 17s ago 17m 357M 1937M 19.0.0-2267-g960fbbe4 19e12293512f 78a229f593f8 2024-03-21T12:48:56.046 INFO:teuthology.orchestra.run.smithi022.stdout:osd.3 smithi022 running (4m) 17s ago 17m 443M 1937M 19.0.0-2267-g960fbbe4 19e12293512f 61a63b67d9af 2024-03-21T12:48:56.046 INFO:teuthology.orchestra.run.smithi022.stdout:osd.4 smithi189 running (4m) 19s ago 17m 370M 2704M 19.0.0-2267-g960fbbe4 19e12293512f 382755cee998 2024-03-21T12:48:56.046 INFO:teuthology.orchestra.run.smithi022.stdout:osd.5 smithi189 running (3m) 19s ago 16m 333M 2704M 19.0.0-2267-g960fbbe4 19e12293512f d2c15ba54b4f 2024-03-21T12:48:56.046 INFO:teuthology.orchestra.run.smithi022.stdout:osd.6 smithi189 running (2m) 19s ago 16m 314M 2704M 19.0.0-2267-g960fbbe4 19e12293512f 291089fb3596 2024-03-21T12:48:56.046 INFO:teuthology.orchestra.run.smithi022.stdout:osd.7 smithi189 running (118s) 19s ago 15m 311M 2704M 19.0.0-2267-g960fbbe4 19e12293512f 247b82310a8a 2024-03-21T12:48:56.046 INFO:teuthology.orchestra.run.smithi022.stdout:prometheus.smithi022 smithi022 *:9095 starting - - - - 2024-03-21T12:48:56.490 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-03-21T12:48:56.490 INFO:teuthology.orchestra.run.smithi022.stdout: "mon": { 2024-03-21T12:48:56.490 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 2 2024-03-21T12:48:56.490 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:48:56.490 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr": { 2024-03-21T12:48:56.490 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 2 2024-03-21T12:48:56.490 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:48:56.490 INFO:teuthology.orchestra.run.smithi022.stdout: "osd": { 2024-03-21T12:48:56.490 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 8 2024-03-21T12:48:56.490 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:48:56.490 INFO:teuthology.orchestra.run.smithi022.stdout: "mds": { 2024-03-21T12:48:56.491 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 2 2024-03-21T12:48:56.491 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:48:56.491 INFO:teuthology.orchestra.run.smithi022.stdout: "overall": { 2024-03-21T12:48:56.491 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 14 2024-03-21T12:48:56.491 INFO:teuthology.orchestra.run.smithi022.stdout: } 2024-03-21T12:48:56.491 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-03-21T12:48:56.847 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-03-21T12:48:56.847 INFO:teuthology.orchestra.run.smithi022.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93", 2024-03-21T12:48:56.847 INFO:teuthology.orchestra.run.smithi022.stdout: "in_progress": true, 2024-03-21T12:48:56.847 INFO:teuthology.orchestra.run.smithi022.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-21T12:48:56.847 INFO:teuthology.orchestra.run.smithi022.stdout: "services_complete": [ 2024-03-21T12:48:56.847 INFO:teuthology.orchestra.run.smithi022.stdout: "mon", 2024-03-21T12:48:56.847 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr", 2024-03-21T12:48:56.847 INFO:teuthology.orchestra.run.smithi022.stdout: "osd", 2024-03-21T12:48:56.848 INFO:teuthology.orchestra.run.smithi022.stdout: "crash", 2024-03-21T12:48:56.848 INFO:teuthology.orchestra.run.smithi022.stdout: "mds", 2024-03-21T12:48:56.848 INFO:teuthology.orchestra.run.smithi022.stdout: "nfs" 2024-03-21T12:48:56.848 INFO:teuthology.orchestra.run.smithi022.stdout: ], 2024-03-21T12:48:56.848 INFO:teuthology.orchestra.run.smithi022.stdout: "progress": "18/23 daemons upgraded", 2024-03-21T12:48:56.848 INFO:teuthology.orchestra.run.smithi022.stdout: "message": "Currently upgrading prometheus daemons", 2024-03-21T12:48:56.848 INFO:teuthology.orchestra.run.smithi022.stdout: "is_paused": false 2024-03-21T12:48:56.848 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-03-21T12:48:57.325 INFO:teuthology.orchestra.run.smithi022.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-03-21T12:48:57.325 INFO:teuthology.orchestra.run.smithi022.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-21T12:48:57.325 INFO:teuthology.orchestra.run.smithi022.stdout: daemon grafana.smithi022 on smithi022 is in error state 2024-03-21T12:48:57.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:57 smithi022 ceph-mon[67047]: pgmap v380: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:48:57.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:57 smithi022 ceph-mon[67047]: from='client.34590 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:48:57.595 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:57 smithi022 ceph-mon[67047]: from='client.34594 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:48:57.595 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:57 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:57.595 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:57 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:57.595 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:57 smithi022 ceph-mon[67047]: from='client.? 172.21.15.22:0/1628237769' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:48:57.595 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:57 smithi022 ceph-mon[67047]: from='client.? 172.21.15.22:0/3222540336' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-21T12:48:57.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:57 smithi189 ceph-mon[53160]: pgmap v380: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:48:57.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:57 smithi189 ceph-mon[53160]: from='client.34590 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:48:57.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:57 smithi189 ceph-mon[53160]: from='client.34594 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:48:57.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:57 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:57.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:57 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:57.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:57 smithi189 ceph-mon[53160]: from='client.? 172.21.15.22:0/1628237769' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:48:57.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:57 smithi189 ceph-mon[53160]: from='client.? 172.21.15.22:0/3222540336' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-21T12:48:58.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:58 smithi022 ceph-mon[67047]: from='client.34602 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:48:58.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:58 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:58.595 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:58 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:58.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:58 smithi189 ceph-mon[53160]: from='client.34602 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:48:58.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:58 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:58.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:58 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:48:59.620 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:48:59 smithi022 ceph-mon[67047]: pgmap v381: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:48:59.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:48:59 smithi189 ceph-mon[53160]: pgmap v381: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:49:00.627 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:00 smithi022 ceph-mon[67047]: pgmap v382: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:49:00.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:00 smithi189 ceph-mon[53160]: pgmap v382: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:49:02.434 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:02 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:49:02.434 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:02 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:49:02.434 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:02 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:49:02.434 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:02 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:49:02.434 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:02 smithi022 ceph-mon[67047]: pgmap v383: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 172 B/s rd, 0 op/s 2024-03-21T12:49:02.434 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:02 smithi022 ceph-mon[67047]: pgmap v384: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 207 B/s rd, 0 op/s 2024-03-21T12:49:02.434 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:02 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:49:02.435 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:02 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:49:02.435 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:02 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-21T12:49:02.435 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:02 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:49:02.435 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:02 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:02.435 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:02 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:02.435 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:02 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:02.435 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:02 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:02.435 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:02 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:02.435 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:02 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:02.435 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:02 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:02.435 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:02 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:02.435 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:02 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:02.435 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:02 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:02.435 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:02 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:02.435 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:02 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:02.435 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:02 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:02.435 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:02 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:02.588 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:02 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:49:02.589 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:02 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:49:02.589 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:02 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:49:02.589 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:02 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:49:02.589 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:02 smithi189 ceph-mon[53160]: pgmap v383: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 172 B/s rd, 0 op/s 2024-03-21T12:49:02.589 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:02 smithi189 ceph-mon[53160]: pgmap v384: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 207 B/s rd, 0 op/s 2024-03-21T12:49:02.589 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:02 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:49:02.589 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:02 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:49:02.589 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:02 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-21T12:49:02.589 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:02 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:49:02.589 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:02 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:02.589 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:02 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:02.589 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:02 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:02.589 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:02 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:02.590 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:02 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:02.590 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:02 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:02.590 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:02 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:02.590 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:02 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:02.590 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:02 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:02.590 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:02 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:02.590 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:02 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:02.590 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:02 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:02.590 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:02 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:02.590 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:02 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:03.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:03 smithi022 ceph-mon[67047]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-21T12:49:03.741 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:03 smithi189 ceph-mon[53160]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-21T12:49:04.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:04 smithi022 ceph-mon[67047]: Upgrade: Updating alertmanager.smithi022 2024-03-21T12:49:04.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:04 smithi022 ceph-mon[67047]: Deploying daemon alertmanager.smithi022 on smithi022 2024-03-21T12:49:04.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:04 smithi022 ceph-mon[67047]: pgmap v385: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 207 B/s rd, 0 op/s 2024-03-21T12:49:04.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:04 smithi189 ceph-mon[53160]: Upgrade: Updating alertmanager.smithi022 2024-03-21T12:49:04.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:04 smithi189 ceph-mon[53160]: Deploying daemon alertmanager.smithi022 on smithi022 2024-03-21T12:49:04.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:04 smithi189 ceph-mon[53160]: pgmap v385: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 207 B/s rd, 0 op/s 2024-03-21T12:49:06.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:06 smithi189 ceph-mon[53160]: pgmap v386: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 207 B/s rd, 0 op/s 2024-03-21T12:49:06.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:06 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:49:06.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:06 smithi022 ceph-mon[67047]: pgmap v386: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 207 B/s rd, 0 op/s 2024-03-21T12:49:06.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:06 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:49:08.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:08 smithi022 ceph-mon[67047]: pgmap v387: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 207 B/s rd, 0 op/s 2024-03-21T12:49:08.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:08 smithi189 ceph-mon[53160]: pgmap v387: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 207 B/s rd, 0 op/s 2024-03-21T12:49:10.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:10 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:49:10.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:10 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:49:10.595 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:10 smithi022 ceph-mon[67047]: pgmap v388: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 207 B/s rd, 0 op/s 2024-03-21T12:49:10.595 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:10 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:49:10.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:10 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:49:10.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:10 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:49:10.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:10 smithi189 ceph-mon[53160]: pgmap v388: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 207 B/s rd, 0 op/s 2024-03-21T12:49:10.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:10 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:49:12.672 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:12 smithi022 ceph-mon[67047]: pgmap v389: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-03-21T12:49:12.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:12 smithi189 ceph-mon[53160]: pgmap v389: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-03-21T12:49:14.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:14 smithi189 ceph-mon[53160]: pgmap v390: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-03-21T12:49:14.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:14 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:49:14.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:14 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:49:14.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:14 smithi022 ceph-mon[67047]: pgmap v390: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-03-21T12:49:14.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:14 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:49:14.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:14 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:49:16.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:16 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:49:16.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:16 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:49:16.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:16 smithi189 ceph-mon[53160]: pgmap v391: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:49:16.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:16 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:49:16.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:16 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:49:16.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:16 smithi022 ceph-mon[67047]: pgmap v391: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:49:18.770 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:18 smithi022 ceph-mon[67047]: pgmap v392: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:49:18.785 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:18 smithi189 ceph-mon[53160]: pgmap v392: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:49:19.844 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:19 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:49:19.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:19 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:49:19.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:19 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:49:19.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:19 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:49:19.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:19 smithi022 ceph-mon[67047]: pgmap v393: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 178 B/s rd, 0 op/s 2024-03-21T12:49:19.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:19 smithi022 ceph-mon[67047]: pgmap v394: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 216 B/s rd, 0 op/s 2024-03-21T12:49:19.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:19 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:49:19.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:19 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:49:19.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:19 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-21T12:49:19.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:19 smithi022 ceph-mon[67047]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-21T12:49:19.845 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:19 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:49:19.846 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:19 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:19.846 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:19 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:19.846 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:19 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:19.846 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:19 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:19.846 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:19 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:19.846 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:19 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:19.846 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:19 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:19.846 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:19 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:19.846 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:19 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:19.846 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:19 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:19.846 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:19 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:19.846 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:19 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:19.847 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:19 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:19.847 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:19 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:19.847 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:19 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:20.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:19 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:49:20.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:19 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:49:20.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:19 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:49:20.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:19 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:49:20.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:19 smithi189 ceph-mon[53160]: pgmap v393: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 178 B/s rd, 0 op/s 2024-03-21T12:49:20.287 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:19 smithi189 ceph-mon[53160]: pgmap v394: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 216 B/s rd, 0 op/s 2024-03-21T12:49:20.287 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:19 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:49:20.287 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:19 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:49:20.287 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:19 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-21T12:49:20.287 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:19 smithi189 ceph-mon[53160]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-21T12:49:20.287 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:19 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:49:20.287 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:19 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:20.287 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:19 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:20.287 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:19 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:20.287 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:19 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:20.287 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:19 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:20.287 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:19 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:20.287 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:19 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:20.287 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:19 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:20.287 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:19 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:20.287 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:19 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:20.288 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:19 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:20.288 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:19 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:20.288 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:19 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:20.288 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:19 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:20.288 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:19 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:21.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:20 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:49:21.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:20 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:49:22.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:21 smithi022 ceph-mon[67047]: Upgrade: Updating grafana.smithi022 2024-03-21T12:49:22.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:21 smithi022 ceph-mon[67047]: Deploying daemon grafana.smithi022 on smithi022 2024-03-21T12:49:22.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:21 smithi022 ceph-mon[67047]: pgmap v395: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 216 B/s rd, 0 op/s 2024-03-21T12:49:22.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:21 smithi189 ceph-mon[53160]: Upgrade: Updating grafana.smithi022 2024-03-21T12:49:22.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:21 smithi189 ceph-mon[53160]: Deploying daemon grafana.smithi022 on smithi022 2024-03-21T12:49:22.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:21 smithi189 ceph-mon[53160]: pgmap v395: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 216 B/s rd, 0 op/s 2024-03-21T12:49:24.068 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:23 smithi022 ceph-mon[67047]: pgmap v396: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-03-21T12:49:24.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:23 smithi189 ceph-mon[53160]: pgmap v396: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-03-21T12:49:26.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:25 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:49:26.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:25 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:49:26.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:25 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:49:26.036 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:25 smithi189 ceph-mon[53160]: pgmap v397: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 216 B/s rd, 0 op/s 2024-03-21T12:49:26.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:25 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:49:26.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:25 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:49:26.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:25 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:49:26.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:25 smithi022 ceph-mon[67047]: pgmap v397: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 216 B/s rd, 0 op/s 2024-03-21T12:49:27.731 INFO:teuthology.orchestra.run.smithi022.stdout:true 2024-03-21T12:49:28.088 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:27 smithi022 ceph-mon[67047]: pgmap v398: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 216 B/s rd, 0 op/s 2024-03-21T12:49:28.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:27 smithi189 ceph-mon[53160]: pgmap v398: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 216 B/s rd, 0 op/s 2024-03-21T12:49:28.433 INFO:teuthology.orchestra.run.smithi022.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-21T12:49:28.434 INFO:teuthology.orchestra.run.smithi022.stdout:alertmanager.smithi022 smithi022 *:9093,9094 running (19s) 14s ago 20m 16.1M - 0.25.0 c8568f914cd2 c5d3945ee326 2024-03-21T12:49:28.434 INFO:teuthology.orchestra.run.smithi022.stdout:crash.smithi022 smithi022 running (8m) 14s ago 20m 6660k - 19.0.0-2267-g960fbbe4 19e12293512f 356b2ddb609a 2024-03-21T12:49:28.434 INFO:teuthology.orchestra.run.smithi022.stdout:crash.smithi189 smithi189 running (8m) 51s ago 20m 6660k - 19.0.0-2267-g960fbbe4 19e12293512f 90ee837f0500 2024-03-21T12:49:28.434 INFO:teuthology.orchestra.run.smithi022.stdout:grafana.smithi022 smithi022 *:3000 starting - - - - 2024-03-21T12:49:28.434 INFO:teuthology.orchestra.run.smithi022.stdout:mds.foofs.smithi022.bcnxho smithi022 running (2m) 14s ago 15m 34.2M - 19.0.0-2267-g960fbbe4 19e12293512f 6db0e27a3822 2024-03-21T12:49:28.434 INFO:teuthology.orchestra.run.smithi022.stdout:mds.foofs.smithi189.lluvko smithi189 running (98s) 51s ago 14m 12.6M - 19.0.0-2267-g960fbbe4 19e12293512f ee1f46135bb3 2024-03-21T12:49:28.434 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.smithi022.fektzs smithi022 *:8443,9283,8765 running (10m) 14s ago 22m 515M - 19.0.0-2267-g960fbbe4 19e12293512f d57cebd306b3 2024-03-21T12:49:28.434 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.smithi189.cnxzvu smithi189 *:8443,9283,8765 running (9m) 51s ago 20m 453M - 19.0.0-2267-g960fbbe4 19e12293512f 41663955c56f 2024-03-21T12:49:28.434 INFO:teuthology.orchestra.run.smithi022.stdout:mon.smithi022 smithi022 running (9m) 14s ago 22m 63.8M 2048M 19.0.0-2267-g960fbbe4 19e12293512f c620c9071b2c 2024-03-21T12:49:28.434 INFO:teuthology.orchestra.run.smithi022.stdout:mon.smithi189 smithi189 running (9m) 51s ago 20m 51.3M 2048M 19.0.0-2267-g960fbbe4 19e12293512f b353af3d8f3b 2024-03-21T12:49:28.434 INFO:teuthology.orchestra.run.smithi022.stdout:nfs.foo.0.0.smithi022.ynhbbq smithi022 *:2049 running (85s) 14s ago 14m 71.9M - 5.7 19e12293512f 825c79cba449 2024-03-21T12:49:28.434 INFO:teuthology.orchestra.run.smithi022.stdout:nfs.foo.1.0.smithi189.yzsqml smithi189 *:2049 running (79s) 51s ago 14m 68.7M - 5.7 19e12293512f f6ea207ea727 2024-03-21T12:49:28.435 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.smithi022 smithi022 *:9100 running (61s) 14s ago 20m 8204k - 1.5.0 0da6a335fe13 a05caa6c4bcb 2024-03-21T12:49:28.435 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.smithi189 smithi189 *:9100 running (55s) 51s ago 19m 5435k - 1.5.0 0da6a335fe13 541a9c8dca57 2024-03-21T12:49:28.435 INFO:teuthology.orchestra.run.smithi022.stdout:osd.0 smithi022 running (7m) 14s ago 19m 407M 1937M 19.0.0-2267-g960fbbe4 19e12293512f 23a85645eeea 2024-03-21T12:49:28.435 INFO:teuthology.orchestra.run.smithi022.stdout:osd.1 smithi022 running (6m) 14s ago 18m 384M 1937M 19.0.0-2267-g960fbbe4 19e12293512f 0f90e70bbf0a 2024-03-21T12:49:28.435 INFO:teuthology.orchestra.run.smithi022.stdout:osd.2 smithi022 running (6m) 14s ago 18m 358M 1937M 19.0.0-2267-g960fbbe4 19e12293512f 78a229f593f8 2024-03-21T12:49:28.435 INFO:teuthology.orchestra.run.smithi022.stdout:osd.3 smithi022 running (5m) 14s ago 18m 444M 1937M 19.0.0-2267-g960fbbe4 19e12293512f 61a63b67d9af 2024-03-21T12:49:28.435 INFO:teuthology.orchestra.run.smithi022.stdout:osd.4 smithi189 running (4m) 51s ago 17m 370M 2704M 19.0.0-2267-g960fbbe4 19e12293512f 382755cee998 2024-03-21T12:49:28.435 INFO:teuthology.orchestra.run.smithi022.stdout:osd.5 smithi189 running (4m) 51s ago 17m 333M 2704M 19.0.0-2267-g960fbbe4 19e12293512f d2c15ba54b4f 2024-03-21T12:49:28.435 INFO:teuthology.orchestra.run.smithi022.stdout:osd.6 smithi189 running (3m) 51s ago 16m 314M 2704M 19.0.0-2267-g960fbbe4 19e12293512f 291089fb3596 2024-03-21T12:49:28.435 INFO:teuthology.orchestra.run.smithi022.stdout:osd.7 smithi189 running (2m) 51s ago 16m 311M 2704M 19.0.0-2267-g960fbbe4 19e12293512f 247b82310a8a 2024-03-21T12:49:28.435 INFO:teuthology.orchestra.run.smithi022.stdout:prometheus.smithi022 smithi022 *:9095 running (37s) 14s ago 19m 52.4M - 2.43.0 a07b618ecd1d 6b4dde428f05 2024-03-21T12:49:28.937 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-03-21T12:49:28.937 INFO:teuthology.orchestra.run.smithi022.stdout: "mon": { 2024-03-21T12:49:28.937 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 2 2024-03-21T12:49:28.937 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:49:28.937 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr": { 2024-03-21T12:49:28.937 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 2 2024-03-21T12:49:28.937 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:49:28.937 INFO:teuthology.orchestra.run.smithi022.stdout: "osd": { 2024-03-21T12:49:28.937 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 8 2024-03-21T12:49:28.937 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:49:28.938 INFO:teuthology.orchestra.run.smithi022.stdout: "mds": { 2024-03-21T12:49:28.938 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 2 2024-03-21T12:49:28.938 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:49:28.938 INFO:teuthology.orchestra.run.smithi022.stdout: "overall": { 2024-03-21T12:49:28.938 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 14 2024-03-21T12:49:28.938 INFO:teuthology.orchestra.run.smithi022.stdout: } 2024-03-21T12:49:28.938 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-03-21T12:49:29.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:28 smithi022 ceph-mon[67047]: from='client.34610 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:49:29.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:28 smithi022 ceph-mon[67047]: from='client.34614 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:49:29.285 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:28 smithi189 ceph-mon[53160]: from='client.34610 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:49:29.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:28 smithi189 ceph-mon[53160]: from='client.34614 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:49:29.317 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-03-21T12:49:29.317 INFO:teuthology.orchestra.run.smithi022.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93", 2024-03-21T12:49:29.317 INFO:teuthology.orchestra.run.smithi022.stdout: "in_progress": true, 2024-03-21T12:49:29.317 INFO:teuthology.orchestra.run.smithi022.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-21T12:49:29.317 INFO:teuthology.orchestra.run.smithi022.stdout: "services_complete": [ 2024-03-21T12:49:29.317 INFO:teuthology.orchestra.run.smithi022.stdout: "mon", 2024-03-21T12:49:29.317 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr", 2024-03-21T12:49:29.317 INFO:teuthology.orchestra.run.smithi022.stdout: "osd", 2024-03-21T12:49:29.317 INFO:teuthology.orchestra.run.smithi022.stdout: "crash", 2024-03-21T12:49:29.317 INFO:teuthology.orchestra.run.smithi022.stdout: "mds", 2024-03-21T12:49:29.317 INFO:teuthology.orchestra.run.smithi022.stdout: "nfs" 2024-03-21T12:49:29.318 INFO:teuthology.orchestra.run.smithi022.stdout: ], 2024-03-21T12:49:29.318 INFO:teuthology.orchestra.run.smithi022.stdout: "progress": "18/23 daemons upgraded", 2024-03-21T12:49:29.318 INFO:teuthology.orchestra.run.smithi022.stdout: "message": "Currently upgrading grafana daemons", 2024-03-21T12:49:29.318 INFO:teuthology.orchestra.run.smithi022.stdout: "is_paused": false 2024-03-21T12:49:29.318 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-03-21T12:49:29.768 INFO:teuthology.orchestra.run.smithi022.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-03-21T12:49:29.768 INFO:teuthology.orchestra.run.smithi022.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-21T12:49:29.768 INFO:teuthology.orchestra.run.smithi022.stdout: daemon grafana.smithi022 on smithi022 is in error state 2024-03-21T12:49:30.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:29 smithi022 ceph-mon[67047]: from='client.34618 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:49:30.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:29 smithi022 ceph-mon[67047]: pgmap v399: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-03-21T12:49:30.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:29 smithi022 ceph-mon[67047]: from='client.? 172.21.15.22:0/10443712' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:30.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:29 smithi022 ceph-mon[67047]: from='client.34626 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:49:30.095 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:29 smithi022 ceph-mon[67047]: from='client.? 172.21.15.22:0/1506523263' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-21T12:49:30.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:29 smithi189 ceph-mon[53160]: from='client.34618 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:49:30.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:29 smithi189 ceph-mon[53160]: pgmap v399: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-03-21T12:49:30.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:29 smithi189 ceph-mon[53160]: from='client.? 172.21.15.22:0/10443712' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:30.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:29 smithi189 ceph-mon[53160]: from='client.34626 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:49:30.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:29 smithi189 ceph-mon[53160]: from='client.? 172.21.15.22:0/1506523263' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-21T12:49:31.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:31 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:49:31.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:31 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:49:31.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:31 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:49:31.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:31 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:49:32.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:32 smithi022 ceph-mon[67047]: pgmap v400: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:49:32.595 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:32 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:49:32.595 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:32 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:49:32.681 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:32 smithi189 ceph-mon[53160]: pgmap v400: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:49:32.682 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:32 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:49:32.682 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:32 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:49:34.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:34 smithi022 ceph-mon[67047]: pgmap v401: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:49:34.786 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:34 smithi189 ceph-mon[53160]: pgmap v401: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:49:36.289 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:35 smithi189 ceph-mon[53160]: pgmap v402: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-03-21T12:49:36.289 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:35 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:49:36.289 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:35 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:49:36.290 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:35 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:49:36.290 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:35 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:49:36.290 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:35 smithi189 ceph-mon[53160]: pgmap v403: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 201 B/s rd, 0 op/s 2024-03-21T12:49:36.290 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:35 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:49:36.290 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:35 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:49:36.290 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:35 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-21T12:49:36.290 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:35 smithi189 ceph-mon[53160]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-21T12:49:36.290 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:35 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:49:36.290 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:35 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:36.290 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:35 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:36.290 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:35 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:36.290 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:35 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:36.290 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:35 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:36.290 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:35 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:36.290 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:35 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:36.291 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:35 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:36.291 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:35 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:36.291 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:35 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:36.291 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:35 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:36.291 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:35 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:36.291 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:35 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:36.291 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:35 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:36.291 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:35 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:36.291 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:35 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:36.291 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:35 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:36.291 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:35 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:36.291 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:35 smithi189 ceph-mon[53160]: Upgrade: Finalizing container_image settings 2024-03-21T12:49:36.291 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:35 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:49:36.291 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:35 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-03-21T12:49:36.291 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:35 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-03-21T12:49:36.291 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:35 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-03-21T12:49:36.291 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:35 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-03-21T12:49:36.292 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:35 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-03-21T12:49:36.292 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:35 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-03-21T12:49:36.292 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:35 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-03-21T12:49:36.292 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:35 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-03-21T12:49:36.292 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:35 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-03-21T12:49:36.292 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:35 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-03-21T12:49:36.292 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:35 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-03-21T12:49:36.292 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:35 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-03-21T12:49:36.292 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:35 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-03-21T12:49:36.292 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:35 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-03-21T12:49:36.292 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:35 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-03-21T12:49:36.292 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:35 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-03-21T12:49:36.292 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:35 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-03-21T12:49:36.292 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:35 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-03-21T12:49:36.292 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:35 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-03-21T12:49:36.293 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:35 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-03-21T12:49:36.293 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:35 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-03-21T12:49:36.293 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:35 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-03-21T12:49:36.293 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:35 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-03-21T12:49:36.293 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:35 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-03-21T12:49:36.293 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:35 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-03-21T12:49:36.293 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:35 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-03-21T12:49:36.293 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:35 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-03-21T12:49:36.293 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:35 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-03-21T12:49:36.293 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:35 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-03-21T12:49:36.293 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:35 smithi189 ceph-mon[53160]: Upgrade: Complete! 2024-03-21T12:49:36.293 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:35 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-03-21T12:49:36.293 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:35 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-03-21T12:49:36.293 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:35 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:49:36.293 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:35 smithi189 ceph-mon[53160]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-03-21T12:49:36.294 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:35 smithi189 ceph-mon[53160]: Cluster is now healthy 2024-03-21T12:49:36.294 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:35 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:49:36.294 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:35 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:49:36.294 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:35 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:49:36.294 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:35 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:49:36.294 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:35 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:49:36.294 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:35 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:49:36.294 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:35 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:49:36.294 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:35 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:49:36.294 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:35 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:49:36.294 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:35 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:49:36.294 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:35 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:49:36.294 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:35 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:49:36.294 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:35 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:49:36.345 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:35 smithi022 ceph-mon[67047]: pgmap v402: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-03-21T12:49:36.345 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:35 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:49:36.345 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:35 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:49:36.345 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:35 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:49:36.345 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:35 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:49:36.345 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:35 smithi022 ceph-mon[67047]: pgmap v403: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 201 B/s rd, 0 op/s 2024-03-21T12:49:36.345 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:35 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:49:36.345 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:35 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:49:36.346 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:35 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-21T12:49:36.346 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:35 smithi022 ceph-mon[67047]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-21T12:49:36.346 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:35 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:49:36.346 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:35 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:36.346 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:35 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:36.346 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:35 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:36.346 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:35 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:36.346 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:35 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:36.346 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:35 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:36.346 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:35 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:36.346 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:35 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:36.346 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:35 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:36.346 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:35 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:36.346 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:35 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:36.347 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:35 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:36.347 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:35 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:36.347 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:35 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:36.347 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:35 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:36.347 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:35 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:36.347 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:35 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:36.347 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:35 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:49:36.347 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:35 smithi022 ceph-mon[67047]: Upgrade: Finalizing container_image settings 2024-03-21T12:49:36.347 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:35 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:49:36.347 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:35 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-03-21T12:49:36.347 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:35 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-03-21T12:49:36.347 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:35 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-03-21T12:49:36.347 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:35 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-03-21T12:49:36.347 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:35 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-03-21T12:49:36.347 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:35 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-03-21T12:49:36.348 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:35 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-03-21T12:49:36.348 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:35 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-03-21T12:49:36.348 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:35 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-03-21T12:49:36.348 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:35 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-03-21T12:49:36.348 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:35 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-03-21T12:49:36.348 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:35 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-03-21T12:49:36.348 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:35 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-03-21T12:49:36.348 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:35 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-03-21T12:49:36.348 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:35 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-03-21T12:49:36.348 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:35 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-03-21T12:49:36.348 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:35 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-03-21T12:49:36.348 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:35 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-03-21T12:49:36.348 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:35 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-03-21T12:49:36.348 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:35 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-03-21T12:49:36.348 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:35 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-03-21T12:49:36.349 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:35 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-03-21T12:49:36.349 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:35 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-03-21T12:49:36.349 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:35 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-03-21T12:49:36.349 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:35 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-03-21T12:49:36.349 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:35 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-03-21T12:49:36.349 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:35 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-03-21T12:49:36.349 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:35 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-03-21T12:49:36.349 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:35 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-03-21T12:49:36.349 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:35 smithi022 ceph-mon[67047]: Upgrade: Complete! 2024-03-21T12:49:36.349 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:35 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-03-21T12:49:36.349 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:35 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-03-21T12:49:36.349 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:35 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:49:36.349 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:35 smithi022 ceph-mon[67047]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-03-21T12:49:36.349 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:35 smithi022 ceph-mon[67047]: Cluster is now healthy 2024-03-21T12:49:36.349 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:35 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:49:36.349 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:35 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:49:36.350 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:35 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:49:36.350 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:35 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:49:36.350 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:35 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:49:36.350 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:35 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:49:36.350 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:35 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:49:36.350 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:35 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:49:36.350 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:35 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:49:36.350 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:35 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:49:36.350 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:35 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:49:36.350 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:35 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:49:36.350 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:35 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:49:37.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:37 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:49:37.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:37 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:49:38.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:38 smithi022 ceph-mon[67047]: pgmap v404: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 201 B/s rd, 0 op/s 2024-03-21T12:49:38.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:38 smithi189 ceph-mon[53160]: pgmap v404: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 201 B/s rd, 0 op/s 2024-03-21T12:49:40.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:40 smithi022 ceph-mon[67047]: pgmap v405: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 201 B/s rd, 0 op/s 2024-03-21T12:49:40.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:40 smithi189 ceph-mon[53160]: pgmap v405: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 201 B/s rd, 0 op/s 2024-03-21T12:49:42.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:42 smithi022 ceph-mon[67047]: pgmap v406: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 201 B/s rd, 0 op/s 2024-03-21T12:49:42.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:42 smithi189 ceph-mon[53160]: pgmap v406: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 201 B/s rd, 0 op/s 2024-03-21T12:49:44.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:44 smithi022 ceph-mon[67047]: pgmap v407: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 403 B/s rd, 0 op/s 2024-03-21T12:49:44.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:44 smithi189 ceph-mon[53160]: pgmap v407: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 403 B/s rd, 0 op/s 2024-03-21T12:49:46.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:46 smithi022 ceph-mon[67047]: pgmap v408: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 201 B/s rd, 0 op/s 2024-03-21T12:49:46.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:46 smithi189 ceph-mon[53160]: pgmap v408: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 201 B/s rd, 0 op/s 2024-03-21T12:49:48.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:48 smithi022 ceph-mon[67047]: pgmap v409: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:49:48.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:48 smithi189 ceph-mon[53160]: pgmap v409: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:49:50.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:50 smithi022 ceph-mon[67047]: pgmap v410: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:49:50.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:50 smithi189 ceph-mon[53160]: pgmap v410: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:49:51.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:51 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:49:51.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:51 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:49:52.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:52 smithi022 ceph-mon[67047]: pgmap v411: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:49:52.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:52 smithi189 ceph-mon[53160]: pgmap v411: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:49:54.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:54 smithi022 ceph-mon[67047]: pgmap v412: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-03-21T12:49:54.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:54 smithi189 ceph-mon[53160]: pgmap v412: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-03-21T12:49:56.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:56 smithi022 ceph-mon[67047]: pgmap v413: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:49:56.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:56 smithi189 ceph-mon[53160]: pgmap v413: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:49:58.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:49:58 smithi022 ceph-mon[67047]: pgmap v414: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:49:58.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:49:58 smithi189 ceph-mon[53160]: pgmap v414: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:50:00.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:50:00 smithi022 ceph-mon[67047]: pgmap v415: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:50:00.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:50:00 smithi022 ceph-mon[67047]: overall HEALTH_OK 2024-03-21T12:50:00.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:50:00 smithi189 ceph-mon[53160]: pgmap v415: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:50:00.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:50:00 smithi189 ceph-mon[53160]: overall HEALTH_OK 2024-03-21T12:50:00.801 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -e sha1=960fbbe4645c2becc8084669be194c95183c7753 -- bash -c 'ceph orch ps' 2024-03-21T12:50:01.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:50:01 smithi022 ceph-mon[67047]: from='client.34634 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:50:01.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:50:01 smithi189 ceph-mon[53160]: from='client.34634 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:50:02.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:50:02 smithi022 ceph-mon[67047]: pgmap v416: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:50:02.437 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:50:02 smithi189 ceph-mon[53160]: pgmap v416: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:50:02.839 INFO:teuthology.orchestra.run.smithi022.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-21T12:50:02.840 INFO:teuthology.orchestra.run.smithi022.stdout:alertmanager.smithi022 smithi022 *:9093,9094 running (54s) 32s ago 21m 16.2M - 0.25.0 c8568f914cd2 c5d3945ee326 2024-03-21T12:50:02.840 INFO:teuthology.orchestra.run.smithi022.stdout:crash.smithi022 smithi022 running (8m) 32s ago 21m 6660k - 19.0.0-2267-g960fbbe4 19e12293512f 356b2ddb609a 2024-03-21T12:50:02.840 INFO:teuthology.orchestra.run.smithi022.stdout:crash.smithi189 smithi189 running (8m) 86s ago 20m 6660k - 19.0.0-2267-g960fbbe4 19e12293512f 90ee837f0500 2024-03-21T12:50:02.840 INFO:teuthology.orchestra.run.smithi022.stdout:grafana.smithi022 smithi022 *:3000 running (38s) 32s ago 21m 51.2M - 9.4.12 2bacad6d85d8 72f35811a230 2024-03-21T12:50:02.840 INFO:teuthology.orchestra.run.smithi022.stdout:mds.foofs.smithi022.bcnxho smithi022 running (2m) 32s ago 15m 34.3M - 19.0.0-2267-g960fbbe4 19e12293512f 6db0e27a3822 2024-03-21T12:50:02.840 INFO:teuthology.orchestra.run.smithi022.stdout:mds.foofs.smithi189.lluvko smithi189 running (2m) 86s ago 15m 12.6M - 19.0.0-2267-g960fbbe4 19e12293512f ee1f46135bb3 2024-03-21T12:50:02.840 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.smithi022.fektzs smithi022 *:8443,9283,8765 running (11m) 32s ago 22m 522M - 19.0.0-2267-g960fbbe4 19e12293512f d57cebd306b3 2024-03-21T12:50:02.840 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.smithi189.cnxzvu smithi189 *:8443,9283,8765 running (10m) 86s ago 20m 453M - 19.0.0-2267-g960fbbe4 19e12293512f 41663955c56f 2024-03-21T12:50:02.840 INFO:teuthology.orchestra.run.smithi022.stdout:mon.smithi022 smithi022 running (10m) 32s ago 22m 64.5M 2048M 19.0.0-2267-g960fbbe4 19e12293512f c620c9071b2c 2024-03-21T12:50:02.840 INFO:teuthology.orchestra.run.smithi022.stdout:mon.smithi189 smithi189 running (9m) 86s ago 20m 51.3M 2048M 19.0.0-2267-g960fbbe4 19e12293512f b353af3d8f3b 2024-03-21T12:50:02.840 INFO:teuthology.orchestra.run.smithi022.stdout:nfs.foo.0.0.smithi022.ynhbbq smithi022 *:2049 running (119s) 32s ago 15m 72.0M - 5.7 19e12293512f 825c79cba449 2024-03-21T12:50:02.840 INFO:teuthology.orchestra.run.smithi022.stdout:nfs.foo.1.0.smithi189.yzsqml smithi189 *:2049 running (113s) 86s ago 15m 68.7M - 5.7 19e12293512f f6ea207ea727 2024-03-21T12:50:02.840 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.smithi022 smithi022 *:9100 running (96s) 32s ago 21m 8304k - 1.5.0 0da6a335fe13 a05caa6c4bcb 2024-03-21T12:50:02.840 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.smithi189 smithi189 *:9100 running (89s) 86s ago 20m 5435k - 1.5.0 0da6a335fe13 541a9c8dca57 2024-03-21T12:50:02.840 INFO:teuthology.orchestra.run.smithi022.stdout:osd.0 smithi022 running (8m) 32s ago 19m 407M 1937M 19.0.0-2267-g960fbbe4 19e12293512f 23a85645eeea 2024-03-21T12:50:02.840 INFO:teuthology.orchestra.run.smithi022.stdout:osd.1 smithi022 running (7m) 32s ago 19m 384M 1937M 19.0.0-2267-g960fbbe4 19e12293512f 0f90e70bbf0a 2024-03-21T12:50:02.840 INFO:teuthology.orchestra.run.smithi022.stdout:osd.2 smithi022 running (6m) 32s ago 19m 358M 1937M 19.0.0-2267-g960fbbe4 19e12293512f 78a229f593f8 2024-03-21T12:50:02.841 INFO:teuthology.orchestra.run.smithi022.stdout:osd.3 smithi022 running (5m) 32s ago 18m 444M 1937M 19.0.0-2267-g960fbbe4 19e12293512f 61a63b67d9af 2024-03-21T12:50:02.841 INFO:teuthology.orchestra.run.smithi022.stdout:osd.4 smithi189 running (5m) 86s ago 18m 370M 2704M 19.0.0-2267-g960fbbe4 19e12293512f 382755cee998 2024-03-21T12:50:02.841 INFO:teuthology.orchestra.run.smithi022.stdout:osd.5 smithi189 running (4m) 86s ago 17m 333M 2704M 19.0.0-2267-g960fbbe4 19e12293512f d2c15ba54b4f 2024-03-21T12:50:02.841 INFO:teuthology.orchestra.run.smithi022.stdout:osd.6 smithi189 running (3m) 86s ago 17m 314M 2704M 19.0.0-2267-g960fbbe4 19e12293512f 291089fb3596 2024-03-21T12:50:02.841 INFO:teuthology.orchestra.run.smithi022.stdout:osd.7 smithi189 running (3m) 86s ago 17m 311M 2704M 19.0.0-2267-g960fbbe4 19e12293512f 247b82310a8a 2024-03-21T12:50:02.841 INFO:teuthology.orchestra.run.smithi022.stdout:prometheus.smithi022 smithi022 *:9095 running (72s) 32s ago 20m 53.4M - 2.43.0 a07b618ecd1d 6b4dde428f05 2024-03-21T12:50:03.493 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -e sha1=960fbbe4645c2becc8084669be194c95183c7753 -- bash -c 'ceph versions' 2024-03-21T12:50:04.345 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:50:04 smithi022 ceph-mon[67047]: from='client.34638 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:50:04.346 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:50:04 smithi022 ceph-mon[67047]: pgmap v417: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-03-21T12:50:04.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:50:04 smithi189 ceph-mon[53160]: from='client.34638 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:50:04.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:50:04 smithi189 ceph-mon[53160]: pgmap v417: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-03-21T12:50:05.571 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-03-21T12:50:05.571 INFO:teuthology.orchestra.run.smithi022.stdout: "mon": { 2024-03-21T12:50:05.571 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 2 2024-03-21T12:50:05.571 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:50:05.571 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr": { 2024-03-21T12:50:05.571 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 2 2024-03-21T12:50:05.571 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:50:05.571 INFO:teuthology.orchestra.run.smithi022.stdout: "osd": { 2024-03-21T12:50:05.572 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 8 2024-03-21T12:50:05.572 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:50:05.572 INFO:teuthology.orchestra.run.smithi022.stdout: "mds": { 2024-03-21T12:50:05.572 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 2 2024-03-21T12:50:05.572 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:50:05.572 INFO:teuthology.orchestra.run.smithi022.stdout: "overall": { 2024-03-21T12:50:05.572 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 14 2024-03-21T12:50:05.572 INFO:teuthology.orchestra.run.smithi022.stdout: } 2024-03-21T12:50:05.572 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-03-21T12:50:06.225 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -e sha1=960fbbe4645c2becc8084669be194c95183c7753 -- bash -c 'echo "wait for servicemap items w/ changing names to refresh"' 2024-03-21T12:50:06.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:50:06 smithi189 ceph-mon[53160]: pgmap v418: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:50:06.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:50:06 smithi189 ceph-mon[53160]: from='client.? 172.21.15.22:0/1947675688' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:50:06.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:50:06 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:50:06.541 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:50:06 smithi022 ceph-mon[67047]: pgmap v418: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:50:06.541 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:50:06 smithi022 ceph-mon[67047]: from='client.? 172.21.15.22:0/1947675688' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:50:06.541 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:50:06 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:50:07.980 INFO:teuthology.orchestra.run.smithi022.stdout:wait for servicemap items w/ changing names to refresh 2024-03-21T12:50:08.534 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:50:08 smithi022 ceph-mon[67047]: pgmap v419: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:50:08.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:50:08 smithi189 ceph-mon[53160]: pgmap v419: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:50:08.556 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -e sha1=960fbbe4645c2becc8084669be194c95183c7753 -- bash -c 'sleep 60' 2024-03-21T12:50:10.197 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:50:10 smithi022 ceph-mon[67047]: pgmap v420: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:50:10.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:50:10 smithi189 ceph-mon[53160]: pgmap v420: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:50:12.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:50:12 smithi189 ceph-mon[53160]: pgmap v421: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:50:12.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:50:12 smithi022 ceph-mon[67047]: pgmap v421: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:50:14.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:50:14 smithi189 ceph-mon[53160]: pgmap v422: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-03-21T12:50:14.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:50:14 smithi022 ceph-mon[67047]: pgmap v422: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-03-21T12:50:16.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:50:16 smithi189 ceph-mon[53160]: pgmap v423: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:50:16.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:50:16 smithi022 ceph-mon[67047]: pgmap v423: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:50:18.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:50:18 smithi189 ceph-mon[53160]: pgmap v424: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:50:18.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:50:18 smithi022 ceph-mon[67047]: pgmap v424: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:50:20.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:50:20 smithi189 ceph-mon[53160]: pgmap v425: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:50:20.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:50:20 smithi022 ceph-mon[67047]: pgmap v425: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:50:21.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:50:21 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:50:21.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:50:21 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:50:22.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:50:22 smithi189 ceph-mon[53160]: pgmap v426: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:50:22.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:50:22 smithi022 ceph-mon[67047]: pgmap v426: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:50:24.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:50:24 smithi189 ceph-mon[53160]: pgmap v427: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-03-21T12:50:24.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:50:24 smithi022 ceph-mon[67047]: pgmap v427: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-03-21T12:50:26.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:50:26 smithi189 ceph-mon[53160]: pgmap v428: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:50:26.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:50:26 smithi022 ceph-mon[67047]: pgmap v428: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:50:28.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:50:28 smithi189 ceph-mon[53160]: pgmap v429: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:50:28.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:50:28 smithi022 ceph-mon[67047]: pgmap v429: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:50:30.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:50:30 smithi189 ceph-mon[53160]: pgmap v430: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:50:30.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:50:30 smithi022 ceph-mon[67047]: pgmap v430: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:50:32.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:50:32 smithi189 ceph-mon[53160]: pgmap v431: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:50:32.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:50:32 smithi022 ceph-mon[67047]: pgmap v431: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:50:34.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:50:34 smithi189 ceph-mon[53160]: pgmap v432: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-03-21T12:50:34.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:50:34 smithi022 ceph-mon[67047]: pgmap v432: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-03-21T12:50:36.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:50:36 smithi189 ceph-mon[53160]: pgmap v433: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:50:36.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:50:36 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:50:36.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:50:36 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:50:36.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:50:36 smithi022 ceph-mon[67047]: pgmap v433: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:50:36.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:50:36 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:50:36.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:50:36 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:50:37.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:50:37 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:50:37.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:50:37 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:50:37.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:50:37 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:50:37.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:50:37 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:50:37.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:50:37 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:50:37.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:50:37 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:50:37.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:50:37 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:50:37.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:50:37 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:50:38.537 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:50:38 smithi189 ceph-mon[53160]: pgmap v434: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:50:38.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:50:38 smithi022 ceph-mon[67047]: pgmap v434: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:50:40.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:50:40 smithi189 ceph-mon[53160]: pgmap v435: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:50:40.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:50:40 smithi022 ceph-mon[67047]: pgmap v435: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:50:42.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:50:42 smithi189 ceph-mon[53160]: pgmap v436: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:50:42.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:50:42 smithi022 ceph-mon[67047]: pgmap v436: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:50:44.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:50:44 smithi189 ceph-mon[53160]: pgmap v437: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-03-21T12:50:44.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:50:44 smithi022 ceph-mon[67047]: pgmap v437: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-03-21T12:50:46.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:50:46 smithi189 ceph-mon[53160]: pgmap v438: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:50:46.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:50:46 smithi022 ceph-mon[67047]: pgmap v438: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:50:48.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:50:48 smithi189 ceph-mon[53160]: pgmap v439: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:50:48.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:50:48 smithi022 ceph-mon[67047]: pgmap v439: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:50:50.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:50:50 smithi189 ceph-mon[53160]: pgmap v440: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:50:50.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:50:50 smithi022 ceph-mon[67047]: pgmap v440: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:50:51.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:50:51 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:50:51.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:50:51 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:50:52.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:50:52 smithi189 ceph-mon[53160]: pgmap v441: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:50:52.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:50:52 smithi022 ceph-mon[67047]: pgmap v441: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:50:54.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:50:54 smithi189 ceph-mon[53160]: pgmap v442: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-03-21T12:50:54.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:50:54 smithi022 ceph-mon[67047]: pgmap v442: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-03-21T12:50:56.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:50:56 smithi189 ceph-mon[53160]: pgmap v443: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:50:56.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:50:56 smithi022 ceph-mon[67047]: pgmap v443: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:50:58.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:50:58 smithi189 ceph-mon[53160]: pgmap v444: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:50:58.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:50:58 smithi022 ceph-mon[67047]: pgmap v444: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:51:00.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:51:00 smithi189 ceph-mon[53160]: pgmap v445: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:51:00.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:51:00 smithi022 ceph-mon[67047]: pgmap v445: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:51:02.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:51:02 smithi189 ceph-mon[53160]: pgmap v446: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:51:02.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:51:02 smithi022 ceph-mon[67047]: pgmap v446: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:51:04.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:51:04 smithi189 ceph-mon[53160]: pgmap v447: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-03-21T12:51:04.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:51:04 smithi022 ceph-mon[67047]: pgmap v447: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-03-21T12:51:06.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:51:06 smithi189 ceph-mon[53160]: pgmap v448: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:51:06.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:51:06 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:51:06.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:51:06 smithi022 ceph-mon[67047]: pgmap v448: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:51:06.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:51:06 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:51:08.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:51:08 smithi189 ceph-mon[53160]: pgmap v449: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:51:08.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:51:08 smithi022 ceph-mon[67047]: pgmap v449: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:51:10.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:51:10 smithi189 ceph-mon[53160]: pgmap v450: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:51:10.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:51:10 smithi022 ceph-mon[67047]: pgmap v450: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:51:10.960 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -e sha1=960fbbe4645c2becc8084669be194c95183c7753 -- bash -c 'ceph orch ps' 2024-03-21T12:51:12.234 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:51:12 smithi022 ceph-mon[67047]: pgmap v451: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:51:12.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:51:12 smithi189 ceph-mon[53160]: pgmap v451: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:51:12.954 INFO:teuthology.orchestra.run.smithi022.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-21T12:51:12.955 INFO:teuthology.orchestra.run.smithi022.stdout:alertmanager.smithi022 smithi022 *:9093,9094 running (2m) 102s ago 22m 16.2M - 0.25.0 c8568f914cd2 c5d3945ee326 2024-03-21T12:51:12.955 INFO:teuthology.orchestra.run.smithi022.stdout:crash.smithi022 smithi022 running (9m) 102s ago 22m 6660k - 19.0.0-2267-g960fbbe4 19e12293512f 356b2ddb609a 2024-03-21T12:51:12.955 INFO:teuthology.orchestra.run.smithi022.stdout:crash.smithi189 smithi189 running (9m) 2m ago 21m 6660k - 19.0.0-2267-g960fbbe4 19e12293512f 90ee837f0500 2024-03-21T12:51:12.955 INFO:teuthology.orchestra.run.smithi022.stdout:grafana.smithi022 smithi022 *:3000 running (108s) 102s ago 22m 51.2M - 9.4.12 2bacad6d85d8 72f35811a230 2024-03-21T12:51:12.955 INFO:teuthology.orchestra.run.smithi022.stdout:mds.foofs.smithi022.bcnxho smithi022 running (3m) 102s ago 16m 34.3M - 19.0.0-2267-g960fbbe4 19e12293512f 6db0e27a3822 2024-03-21T12:51:12.955 INFO:teuthology.orchestra.run.smithi022.stdout:mds.foofs.smithi189.lluvko smithi189 running (3m) 2m ago 16m 12.6M - 19.0.0-2267-g960fbbe4 19e12293512f ee1f46135bb3 2024-03-21T12:51:12.955 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.smithi022.fektzs smithi022 *:8443,9283,8765 running (12m) 102s ago 23m 522M - 19.0.0-2267-g960fbbe4 19e12293512f d57cebd306b3 2024-03-21T12:51:12.955 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.smithi189.cnxzvu smithi189 *:8443,9283,8765 running (11m) 2m ago 21m 453M - 19.0.0-2267-g960fbbe4 19e12293512f 41663955c56f 2024-03-21T12:51:12.955 INFO:teuthology.orchestra.run.smithi022.stdout:mon.smithi022 smithi022 running (11m) 102s ago 24m 64.5M 2048M 19.0.0-2267-g960fbbe4 19e12293512f c620c9071b2c 2024-03-21T12:51:12.955 INFO:teuthology.orchestra.run.smithi022.stdout:mon.smithi189 smithi189 running (10m) 2m ago 21m 51.3M 2048M 19.0.0-2267-g960fbbe4 19e12293512f b353af3d8f3b 2024-03-21T12:51:12.955 INFO:teuthology.orchestra.run.smithi022.stdout:nfs.foo.0.0.smithi022.ynhbbq smithi022 *:2049 running (3m) 102s ago 16m 72.0M - 5.7 19e12293512f 825c79cba449 2024-03-21T12:51:12.955 INFO:teuthology.orchestra.run.smithi022.stdout:nfs.foo.1.0.smithi189.yzsqml smithi189 *:2049 running (3m) 2m ago 16m 68.7M - 5.7 19e12293512f f6ea207ea727 2024-03-21T12:51:12.955 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.smithi022 smithi022 *:9100 running (2m) 102s ago 22m 8304k - 1.5.0 0da6a335fe13 a05caa6c4bcb 2024-03-21T12:51:12.956 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.smithi189 smithi189 *:9100 running (2m) 2m ago 21m 5435k - 1.5.0 0da6a335fe13 541a9c8dca57 2024-03-21T12:51:12.956 INFO:teuthology.orchestra.run.smithi022.stdout:osd.0 smithi022 running (9m) 102s ago 21m 407M 1937M 19.0.0-2267-g960fbbe4 19e12293512f 23a85645eeea 2024-03-21T12:51:12.956 INFO:teuthology.orchestra.run.smithi022.stdout:osd.1 smithi022 running (8m) 102s ago 20m 384M 1937M 19.0.0-2267-g960fbbe4 19e12293512f 0f90e70bbf0a 2024-03-21T12:51:12.956 INFO:teuthology.orchestra.run.smithi022.stdout:osd.2 smithi022 running (7m) 102s ago 20m 358M 1937M 19.0.0-2267-g960fbbe4 19e12293512f 78a229f593f8 2024-03-21T12:51:12.956 INFO:teuthology.orchestra.run.smithi022.stdout:osd.3 smithi022 running (7m) 102s ago 19m 444M 1937M 19.0.0-2267-g960fbbe4 19e12293512f 61a63b67d9af 2024-03-21T12:51:12.956 INFO:teuthology.orchestra.run.smithi022.stdout:osd.4 smithi189 running (6m) 2m ago 19m 370M 2704M 19.0.0-2267-g960fbbe4 19e12293512f 382755cee998 2024-03-21T12:51:12.956 INFO:teuthology.orchestra.run.smithi022.stdout:osd.5 smithi189 running (5m) 2m ago 18m 333M 2704M 19.0.0-2267-g960fbbe4 19e12293512f d2c15ba54b4f 2024-03-21T12:51:12.956 INFO:teuthology.orchestra.run.smithi022.stdout:osd.6 smithi189 running (4m) 2m ago 18m 314M 2704M 19.0.0-2267-g960fbbe4 19e12293512f 291089fb3596 2024-03-21T12:51:12.956 INFO:teuthology.orchestra.run.smithi022.stdout:osd.7 smithi189 running (4m) 2m ago 18m 311M 2704M 19.0.0-2267-g960fbbe4 19e12293512f 247b82310a8a 2024-03-21T12:51:12.956 INFO:teuthology.orchestra.run.smithi022.stdout:prometheus.smithi022 smithi022 *:9095 running (2m) 102s ago 21m 53.4M - 2.43.0 a07b618ecd1d 6b4dde428f05 2024-03-21T12:51:13.680 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -e sha1=960fbbe4645c2becc8084669be194c95183c7753 -- bash -c 'ceph orch upgrade status' 2024-03-21T12:51:14.302 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:51:14 smithi022 ceph-mon[67047]: from='client.34646 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:51:14.303 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:51:14 smithi022 ceph-mon[67047]: pgmap v452: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-03-21T12:51:14.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:51:14 smithi189 ceph-mon[53160]: from='client.34646 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:51:14.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:51:14 smithi189 ceph-mon[53160]: pgmap v452: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-03-21T12:51:15.662 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-03-21T12:51:15.662 INFO:teuthology.orchestra.run.smithi022.stdout: "target_image": null, 2024-03-21T12:51:15.662 INFO:teuthology.orchestra.run.smithi022.stdout: "in_progress": false, 2024-03-21T12:51:15.662 INFO:teuthology.orchestra.run.smithi022.stdout: "which": "", 2024-03-21T12:51:15.662 INFO:teuthology.orchestra.run.smithi022.stdout: "services_complete": [], 2024-03-21T12:51:15.663 INFO:teuthology.orchestra.run.smithi022.stdout: "progress": null, 2024-03-21T12:51:15.663 INFO:teuthology.orchestra.run.smithi022.stdout: "message": "", 2024-03-21T12:51:15.663 INFO:teuthology.orchestra.run.smithi022.stdout: "is_paused": false 2024-03-21T12:51:15.663 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-03-21T12:51:16.253 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -e sha1=960fbbe4645c2becc8084669be194c95183c7753 -- bash -c 'ceph health detail' 2024-03-21T12:51:16.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:51:16 smithi022 ceph-mon[67047]: pgmap v453: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:51:16.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:51:16 smithi189 ceph-mon[53160]: pgmap v453: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:51:17.454 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:51:17 smithi022 ceph-mon[67047]: from='client.34650 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:51:17.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:51:17 smithi189 ceph-mon[53160]: from='client.34650 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:51:18.256 INFO:teuthology.orchestra.run.smithi022.stdout:HEALTH_OK 2024-03-21T12:51:18.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:51:18 smithi189 ceph-mon[53160]: pgmap v454: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:51:18.578 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:51:18 smithi022 ceph-mon[67047]: pgmap v454: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:51:18.892 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -e sha1=960fbbe4645c2becc8084669be194c95183c7753 -- bash -c 'ceph versions' 2024-03-21T12:51:19.524 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:51:19 smithi022 ceph-mon[67047]: from='client.? 172.21.15.22:0/1224135448' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-21T12:51:19.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:51:19 smithi189 ceph-mon[53160]: from='client.? 172.21.15.22:0/1224135448' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-21T12:51:20.459 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:51:20 smithi022 ceph-mon[67047]: pgmap v455: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:51:20.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:51:20 smithi189 ceph-mon[53160]: pgmap v455: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:51:21.062 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-03-21T12:51:21.062 INFO:teuthology.orchestra.run.smithi022.stdout: "mon": { 2024-03-21T12:51:21.062 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 2 2024-03-21T12:51:21.063 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:51:21.063 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr": { 2024-03-21T12:51:21.063 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 2 2024-03-21T12:51:21.063 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:51:21.063 INFO:teuthology.orchestra.run.smithi022.stdout: "osd": { 2024-03-21T12:51:21.063 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 8 2024-03-21T12:51:21.063 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:51:21.063 INFO:teuthology.orchestra.run.smithi022.stdout: "mds": { 2024-03-21T12:51:21.063 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 2 2024-03-21T12:51:21.063 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:51:21.063 INFO:teuthology.orchestra.run.smithi022.stdout: "overall": { 2024-03-21T12:51:21.063 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)": 14 2024-03-21T12:51:21.064 INFO:teuthology.orchestra.run.smithi022.stdout: } 2024-03-21T12:51:21.064 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-03-21T12:51:21.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:51:21 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:51:21.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:51:21 smithi022 ceph-mon[67047]: from='client.? 172.21.15.22:0/773122809' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:51:21.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:51:21 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:51:21.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:51:21 smithi189 ceph-mon[53160]: from='client.? 172.21.15.22:0/773122809' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:51:21.668 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -e sha1=960fbbe4645c2becc8084669be194c95183c7753 -- bash -c 'ceph versions | jq -e '"'"'.overall | length == 1'"'"'' 2024-03-21T12:51:22.240 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:51:22 smithi022 ceph-mon[67047]: pgmap v456: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:51:22.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:51:22 smithi189 ceph-mon[53160]: pgmap v456: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:51:23.728 INFO:teuthology.orchestra.run.smithi022.stdout:true 2024-03-21T12:51:24.342 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:51:24 smithi022 ceph-mon[67047]: pgmap v457: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-03-21T12:51:24.343 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:51:24 smithi022 ceph-mon[67047]: from='client.? 172.21.15.22:0/948315773' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:51:24.365 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -e sha1=960fbbe4645c2becc8084669be194c95183c7753 -- bash -c 'ceph versions | jq -e '"'"'.overall | keys'"'"' | grep $sha1' 2024-03-21T12:51:24.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:51:24 smithi189 ceph-mon[53160]: pgmap v457: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-03-21T12:51:24.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:51:24 smithi189 ceph-mon[53160]: from='client.? 172.21.15.22:0/948315773' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:51:26.514 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.0.0-2267-g960fbbe4 (960fbbe4645c2becc8084669be194c95183c7753) squid (dev)" 2024-03-21T12:51:26.515 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:51:26 smithi022 ceph-mon[67047]: pgmap v458: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:51:26.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:51:26 smithi189 ceph-mon[53160]: pgmap v458: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:51:27.116 INFO:teuthology.task.sequential:In sequential, running task cephadm.wait_for_service... 2024-03-21T12:51:27.117 INFO:tasks.cephadm:Waiting for ceph service nfs.foo to start (timeout 300)... 2024-03-21T12:51:27.117 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -- ceph orch ls -f json 2024-03-21T12:51:27.428 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:51:27 smithi022 ceph-mon[67047]: from='client.? 172.21.15.22:0/584543661' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:51:27.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:51:27 smithi189 ceph-mon[53160]: from='client.? 172.21.15.22:0/584543661' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-21T12:51:28.398 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:51:28 smithi022 ceph-mon[67047]: pgmap v459: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:51:28.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:51:28 smithi189 ceph-mon[53160]: pgmap v459: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:51:29.161 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-03-21T12:51:29.161 INFO:teuthology.orchestra.run.smithi022.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-03-21T12:28:01.307182Z", "last_refresh": "2024-03-21T12:49:30.185651Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-03-21T12:27:55.198852Z", "last_refresh": "2024-03-21T12:48:36.627660Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "spec": {"anonymous_access": true, "protocol": "https"}, "status": {"created": "2024-03-21T12:27:58.473916Z", "last_refresh": "2024-03-21T12:49:30.185805Z", "ports": [3000], "running": 1, "size": 1}}, {"placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-03-21T12:34:23.013720Z", "last_refresh": "2024-03-21T12:48:36.628320Z", "running": 2, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-03-21T12:27:53.563690Z", "last_refresh": "2024-03-21T12:48:36.627762Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi022:172.21.15.22=smithi022", "smithi189:172.21.15.189=smithi189"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-03-21T12:28:45.410421Z", "last_refresh": "2024-03-21T12:48:36.627843Z", "running": 2, "size": 2}}, {"events": ["2024-03-21T12:50:36.613132Z service:nfs.foo [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foo", "service_name": "nfs.foo", "service_type": "nfs", "spec": {"port": 2049}, "status": {"created": "2024-03-21T12:34:43.945896Z", "last_refresh": "2024-03-21T12:48:36.628400Z", "ports": [2049], "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-03-21T12:27:59.661651Z", "last_refresh": "2024-03-21T12:48:36.627922Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "19e12293512ff7978c8adb5f72a829060dd47f07bfd6903b6b3178d2ebc29e4e", "container_image_name": "quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93", "last_refresh": "2024-03-21T12:48:36.627999Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-03-21T12:27:56.660150Z", "last_refresh": "2024-03-21T12:49:30.185954Z", "ports": [9095], "running": 1, "size": 1}}] 2024-03-21T12:51:29.832 INFO:tasks.cephadm:nfs.foo has 2/2 2024-03-21T12:51:29.832 DEBUG:teuthology.parallel:result is None 2024-03-21T12:51:29.832 INFO:teuthology.run_tasks:Running task vip.exec... 2024-03-21T12:51:29.842 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi022.front.sepia.ceph.com 2024-03-21T12:51:29.843 DEBUG:teuthology.orchestra.run.smithi022:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'umount /mnt/foo' 2024-03-21T12:51:29.871 INFO:teuthology.orchestra.run.smithi022.stderr:+ umount /mnt/foo 2024-03-21T12:51:29.884 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-03-21T12:51:29.893 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi022.front.sepia.ceph.com 2024-03-21T12:51:29.893 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -- bash -c 'ceph nfs cluster ls | grep foo' 2024-03-21T12:51:30.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:51:29 smithi022 ceph-mon[67047]: pgmap v460: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:51:30.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:51:29 smithi022 ceph-mon[67047]: from='client.34670 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-21T12:51:30.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:51:29 smithi189 ceph-mon[53160]: pgmap v460: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:51:30.286 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:51:29 smithi189 ceph-mon[53160]: from='client.34670 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-21T12:51:31.942 INFO:teuthology.orchestra.run.smithi022.stdout: "foo" 2024-03-21T12:51:32.219 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:51:32 smithi022 ceph-mon[67047]: pgmap v461: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:51:32.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:51:32 smithi189 ceph-mon[53160]: pgmap v461: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:51:32.551 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -- bash -c 'ceph nfs export ls foo --detailed' 2024-03-21T12:51:33.139 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:51:33 smithi022 ceph-mon[67047]: from='client.34674 -' entity='client.admin' cmd=[{"prefix": "nfs cluster ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:51:33.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:51:33 smithi189 ceph-mon[53160]: from='client.34674 -' entity='client.admin' cmd=[{"prefix": "nfs cluster ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:51:34.089 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:51:34 smithi022 ceph-mon[67047]: pgmap v462: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-03-21T12:51:34.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:51:34 smithi189 ceph-mon[53160]: pgmap v462: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-03-21T12:51:34.621 INFO:teuthology.orchestra.run.smithi022.stdout:[ 2024-03-21T12:51:34.621 INFO:teuthology.orchestra.run.smithi022.stdout: { 2024-03-21T12:51:34.621 INFO:teuthology.orchestra.run.smithi022.stdout: "access_type": "RW", 2024-03-21T12:51:34.622 INFO:teuthology.orchestra.run.smithi022.stdout: "clients": [], 2024-03-21T12:51:34.622 INFO:teuthology.orchestra.run.smithi022.stdout: "cluster_id": "foo", 2024-03-21T12:51:34.622 INFO:teuthology.orchestra.run.smithi022.stdout: "export_id": 1, 2024-03-21T12:51:34.622 INFO:teuthology.orchestra.run.smithi022.stdout: "fsal": { 2024-03-21T12:51:34.622 INFO:teuthology.orchestra.run.smithi022.stdout: "fs_name": "foofs", 2024-03-21T12:51:34.622 INFO:teuthology.orchestra.run.smithi022.stdout: "name": "CEPH", 2024-03-21T12:51:34.622 INFO:teuthology.orchestra.run.smithi022.stdout: "user_id": "nfs.foo.1" 2024-03-21T12:51:34.622 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-03-21T12:51:34.622 INFO:teuthology.orchestra.run.smithi022.stdout: "path": "/", 2024-03-21T12:51:34.622 INFO:teuthology.orchestra.run.smithi022.stdout: "protocols": [ 2024-03-21T12:51:34.622 INFO:teuthology.orchestra.run.smithi022.stdout: 4 2024-03-21T12:51:34.622 INFO:teuthology.orchestra.run.smithi022.stdout: ], 2024-03-21T12:51:34.622 INFO:teuthology.orchestra.run.smithi022.stdout: "pseudo": "/fake", 2024-03-21T12:51:34.622 INFO:teuthology.orchestra.run.smithi022.stdout: "security_label": true, 2024-03-21T12:51:34.622 INFO:teuthology.orchestra.run.smithi022.stdout: "squash": "none", 2024-03-21T12:51:34.623 INFO:teuthology.orchestra.run.smithi022.stdout: "transports": [ 2024-03-21T12:51:34.623 INFO:teuthology.orchestra.run.smithi022.stdout: "TCP" 2024-03-21T12:51:34.623 INFO:teuthology.orchestra.run.smithi022.stdout: ] 2024-03-21T12:51:34.623 INFO:teuthology.orchestra.run.smithi022.stdout: } 2024-03-21T12:51:34.623 INFO:teuthology.orchestra.run.smithi022.stdout:] 2024-03-21T12:51:35.220 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -- bash -c 'rados -p .nfs --all ls -' 2024-03-21T12:51:36.225 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:51:36 smithi022 ceph-mon[67047]: from='client.34678 -' entity='client.admin' cmd=[{"prefix": "nfs export ls", "cluster_id": "foo", "detailed": true, "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:51:36.225 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:51:36 smithi022 ceph-mon[67047]: pgmap v463: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:51:36.226 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:51:36 smithi022 ceph-mon[67047]: mgrmap e43: smithi022.fektzs(active, since 11m), standbys: smithi189.cnxzvu 2024-03-21T12:51:36.226 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:51:36 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:51:36.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:51:36 smithi189 ceph-mon[53160]: from='client.34678 -' entity='client.admin' cmd=[{"prefix": "nfs export ls", "cluster_id": "foo", "detailed": true, "target": ["mon-mgr", ""]}]: dispatch 2024-03-21T12:51:36.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:51:36 smithi189 ceph-mon[53160]: pgmap v463: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-21T12:51:36.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:51:36 smithi189 ceph-mon[53160]: mgrmap e43: smithi022.fektzs(active, since 11m), standbys: smithi189.cnxzvu 2024-03-21T12:51:36.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:51:36 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-21T12:51:36.995 INFO:teuthology.orchestra.run.smithi022.stdout:foo rec-0000000000000004:nfs.foo.1 2024-03-21T12:51:36.996 INFO:teuthology.orchestra.run.smithi022.stdout:foo grace 2024-03-21T12:51:36.996 INFO:teuthology.orchestra.run.smithi022.stdout:foo rec-0000000000000004:nfs.foo.0 2024-03-21T12:51:36.996 INFO:teuthology.orchestra.run.smithi022.stdout:foo conf-nfs.foo 2024-03-21T12:51:36.996 INFO:teuthology.orchestra.run.smithi022.stdout:foo export-1 2024-03-21T12:51:37.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:51:37 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:51:37.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:51:37 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-21T12:51:37.678 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-03-21T12:51:37.688 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi022.front.sepia.ceph.com 2024-03-21T12:51:37.689 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 -- bash -c 'set -ex 2024-03-21T12:51:37.689 DEBUG:teuthology.orchestra.run.smithi022:> [[ `ceph config get mgr mgr/cephadm/migration_current` -gt 2 ]] 2024-03-21T12:51:37.689 DEBUG:teuthology.orchestra.run.smithi022:> ' 2024-03-21T12:51:38.303 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:51:38 smithi022 ceph-mon[67047]: pgmap v464: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-03-21T12:51:38.303 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:51:38 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:51:38.303 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:51:38 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:51:38.303 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:51:38 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:51:38.304 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:51:38 smithi022 ceph-mon[67047]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:51:38.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:51:38 smithi189 ceph-mon[53160]: pgmap v464: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-03-21T12:51:38.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:51:38 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-21T12:51:38.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:51:38 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-21T12:51:38.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:51:38 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:51:38.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:51:38 smithi189 ceph-mon[53160]: from='mgr.34102 172.21.15.22:0/3443379810' entity='mgr.smithi022.fektzs' 2024-03-21T12:51:39.445 INFO:teuthology.orchestra.run.smithi022.stderr:++ ceph config get mgr mgr/cephadm/migration_current 2024-03-21T12:51:39.708 INFO:teuthology.orchestra.run.smithi022.stderr:+ [[ 6 -gt 2 ]] 2024-03-21T12:51:40.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:51:40 smithi022 ceph-mon[67047]: pgmap v465: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-03-21T12:51:40.344 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:51:40 smithi022 ceph-mon[67047]: from='client.? 172.21.15.22:0/427272838' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/cephadm/migration_current"}]: dispatch 2024-03-21T12:51:40.397 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-03-21T12:51:40.407 INFO:tasks.cephadm:Teardown begin 2024-03-21T12:51:40.408 DEBUG:teuthology.orchestra.run.smithi022:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-03-21T12:51:40.436 DEBUG:teuthology.orchestra.run.smithi189:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-03-21T12:51:40.467 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:51:40 smithi189 ceph-mon[53160]: pgmap v465: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-03-21T12:51:40.467 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:51:40 smithi189 ceph-mon[53160]: from='client.? 172.21.15.22:0/427272838' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/cephadm/migration_current"}]: dispatch 2024-03-21T12:51:40.472 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-03-21T12:51:40.472 DEBUG:teuthology.orchestra.run.smithi022:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-03-21T12:51:40.491 DEBUG:teuthology.orchestra.run.smithi189:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-03-21T12:51:40.529 INFO:tasks.cephadm:Stopping all daemons... 2024-03-21T12:51:40.529 INFO:tasks.cephadm.mon.smithi022:Stopping mon.smithi022... 2024-03-21T12:51:40.530 DEBUG:teuthology.orchestra.run.smithi022:> sudo systemctl stop ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715@mon.smithi022 2024-03-21T12:51:40.836 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:51:40 smithi022 systemd[1]: Stopping Ceph mon.smithi022 for 3fc7f0f2-e77e-11ee-95cd-87774f69a715... 2024-03-21T12:51:41.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:51:40 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-mon-smithi022[67043]: 2024-03-21T12:51:40.835+0000 7f98de7d5700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi022 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-03-21T12:51:41.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:51:40 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-mon-smithi022[67043]: 2024-03-21T12:51:40.835+0000 7f98de7d5700 -1 mon.smithi022@0(leader) e3 *** Got Signal Terminated *** 2024-03-21T12:51:41.094 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:51:40 smithi022 podman[112747]: 2024-03-21 12:51:40.876937609 +0000 UTC m=+0.143168872 container died c620c9071b2c5e4c45ef86ebd2c1e7f031673bb2ec8844d2cdb9d9b86d552e78 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-mon-smithi022, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240319, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30) 2024-03-21T12:51:41.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:51:41 smithi022 podman[112747]: 2024-03-21 12:51:41.436288894 +0000 UTC m=+0.702520156 container cleanup c620c9071b2c5e4c45ef86ebd2c1e7f031673bb2ec8844d2cdb9d9b86d552e78 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-mon-smithi022, org.label-schema.license=GPLv2, org.label-schema.build-date=20240319, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe) 2024-03-21T12:51:41.594 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:51:41 smithi022 bash[112747]: ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-mon-smithi022 2024-03-21T12:51:41.859 DEBUG:teuthology.orchestra.run.smithi022:> sudo pkill -f 'journalctl -f -n 0 -u ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715@mon.smithi022.service' 2024-03-21T12:51:41.884 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:51:41 smithi022 podman[112772]: 2024-03-21 12:51:41.620969302 +0000 UTC m=+0.739882081 container remove c620c9071b2c5e4c45ef86ebd2c1e7f031673bb2ec8844d2cdb9d9b86d552e78 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-mon-smithi022, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240319) 2024-03-21T12:51:41.884 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:51:41 smithi022 systemd[1]: ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715@mon.smithi022.service: Deactivated successfully. 2024-03-21T12:51:41.884 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:51:41 smithi022 systemd[1]: Stopped Ceph mon.smithi022 for 3fc7f0f2-e77e-11ee-95cd-87774f69a715. 2024-03-21T12:51:41.884 INFO:journalctl@ceph.mon.smithi022.smithi022.stdout:Mar 21 12:51:41 smithi022 systemd[1]: ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715@mon.smithi022.service: Consumed 15.111s CPU time. 2024-03-21T12:51:41.917 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-21T12:51:41.917 INFO:tasks.cephadm.mon.smithi022:Stopped mon.smithi022 2024-03-21T12:51:41.918 INFO:tasks.cephadm.mon.smithi189:Stopping mon.smithi189... 2024-03-21T12:51:41.918 DEBUG:teuthology.orchestra.run.smithi189:> sudo systemctl stop ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715@mon.smithi189 2024-03-21T12:51:42.256 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:51:41 smithi189 systemd[1]: Stopping Ceph mon.smithi189 for 3fc7f0f2-e77e-11ee-95cd-87774f69a715... 2024-03-21T12:51:42.535 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:51:42 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-mon-smithi189[53156]: 2024-03-21T12:51:42.256+0000 7f3b65201700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi189 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-03-21T12:51:42.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:51:42 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-mon-smithi189[53156]: 2024-03-21T12:51:42.256+0000 7f3b65201700 -1 mon.smithi189@1(peon) e3 *** Got Signal Terminated *** 2024-03-21T12:51:42.536 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:51:42 smithi189 podman[79255]: 2024-03-21 12:51:42.277244678 +0000 UTC m=+0.156570882 container died b353af3d8f3b6abef504d9964905f2616d9f4e964d160c9c7924d0d1ff28e456 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-mon-smithi189, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240319, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe) 2024-03-21T12:51:43.008 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:51:42 smithi189 podman[79255]: 2024-03-21 12:51:42.76568977 +0000 UTC m=+0.645016028 container cleanup b353af3d8f3b6abef504d9964905f2616d9f4e964d160c9c7924d0d1ff28e456 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-mon-smithi189, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, io.buildah.version=1.33.5, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240319, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.license=GPLv2) 2024-03-21T12:51:43.009 INFO:journalctl@ceph.mon.smithi189.smithi189.stdout:Mar 21 12:51:42 smithi189 bash[79255]: ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-mon-smithi189 2024-03-21T12:51:43.128 DEBUG:teuthology.orchestra.run.smithi189:> sudo pkill -f 'journalctl -f -n 0 -u ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715@mon.smithi189.service' 2024-03-21T12:51:43.177 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-21T12:51:43.177 INFO:tasks.cephadm.mon.smithi189:Stopped mon.smithi189 2024-03-21T12:51:43.177 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-03-21T12:51:43.177 DEBUG:teuthology.orchestra.run.smithi022:> sudo systemctl stop ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715@osd.0 2024-03-21T12:51:43.491 INFO:journalctl@ceph.osd.0.smithi022.stdout:Mar 21 12:51:43 smithi022 systemd[1]: Stopping Ceph osd.0 for 3fc7f0f2-e77e-11ee-95cd-87774f69a715... 2024-03-21T12:51:43.844 INFO:journalctl@ceph.osd.0.smithi022.stdout:Mar 21 12:51:43 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-0[73106]: 2024-03-21T12:51:43.490+0000 7f7cb5966700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-21T12:51:43.844 INFO:journalctl@ceph.osd.0.smithi022.stdout:Mar 21 12:51:43 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-0[73106]: 2024-03-21T12:51:43.490+0000 7f7cb5966700 -1 osd.0 164 *** Got signal Terminated *** 2024-03-21T12:51:43.844 INFO:journalctl@ceph.osd.0.smithi022.stdout:Mar 21 12:51:43 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-0[73106]: 2024-03-21T12:51:43.490+0000 7f7cb5966700 -1 osd.0 164 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-21T12:51:49.282 INFO:journalctl@ceph.osd.0.smithi022.stdout:Mar 21 12:51:48 smithi022 podman[112878]: 2024-03-21 12:51:48.966440608 +0000 UTC m=+5.588912428 container died 23a85645eeea64275ee72959e9c651764187c829fa448bc8a3633f97238da164 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240319, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, org.label-schema.license=GPLv2) 2024-03-21T12:51:49.591 INFO:journalctl@ceph.osd.0.smithi022.stdout:Mar 21 12:51:49 smithi022 podman[112878]: 2024-03-21 12:51:49.382066716 +0000 UTC m=+6.004538528 container cleanup 23a85645eeea64275ee72959e9c651764187c829fa448bc8a3633f97238da164 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-0, GIT_CLEAN=True, ceph=True, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240319, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-03-21T12:51:49.591 INFO:journalctl@ceph.osd.0.smithi022.stdout:Mar 21 12:51:49 smithi022 bash[112878]: ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-0 2024-03-21T12:51:49.844 INFO:journalctl@ceph.osd.0.smithi022.stdout:Mar 21 12:51:49 smithi022 podman[112896]: 2024-03-21 12:51:49.591805948 +0000 UTC m=+0.628999605 container remove 23a85645eeea64275ee72959e9c651764187c829fa448bc8a3633f97238da164 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240319, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, ceph=True, org.label-schema.vendor=CentOS) 2024-03-21T12:51:50.296 INFO:journalctl@ceph.osd.0.smithi022.stdout:Mar 21 12:51:50 smithi022 podman[112987]: 2024-03-21 12:51:49.912340109 +0000 UTC m=+0.018097563 image pull 19e12293512ff7978c8adb5f72a829060dd47f07bfd6903b6b3178d2ebc29e4e quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93 2024-03-21T12:51:50.296 INFO:journalctl@ceph.osd.0.smithi022.stdout:Mar 21 12:51:50 smithi022 podman[112987]: 2024-03-21 12:51:50.026472075 +0000 UTC m=+0.132229554 container create 8a60e94f91184006fde9ff85f55df701ba1c48f4f52c1f895f30fbb37933fed7 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-0-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, org.label-schema.license=GPLv2, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240319, GIT_CLEAN=True) 2024-03-21T12:51:50.594 INFO:journalctl@ceph.osd.0.smithi022.stdout:Mar 21 12:51:50 smithi022 podman[112987]: 2024-03-21 12:51:50.296207792 +0000 UTC m=+0.401965240 container init 8a60e94f91184006fde9ff85f55df701ba1c48f4f52c1f895f30fbb37933fed7 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-0-deactivate, org.label-schema.schema-version=1.0, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240319, io.buildah.version=1.33.5, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-03-21T12:51:50.595 INFO:journalctl@ceph.osd.0.smithi022.stdout:Mar 21 12:51:50 smithi022 podman[112987]: 2024-03-21 12:51:50.301276013 +0000 UTC m=+0.407033458 container start 8a60e94f91184006fde9ff85f55df701ba1c48f4f52c1f895f30fbb37933fed7 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-0-deactivate, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, io.buildah.version=1.33.5, org.label-schema.build-date=20240319, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.license=GPLv2) 2024-03-21T12:51:50.595 INFO:journalctl@ceph.osd.0.smithi022.stdout:Mar 21 12:51:50 smithi022 podman[112987]: 2024-03-21 12:51:50.395951829 +0000 UTC m=+0.501709293 container attach 8a60e94f91184006fde9ff85f55df701ba1c48f4f52c1f895f30fbb37933fed7 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-0-deactivate, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.build-date=20240319, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-03-21T12:51:50.932 INFO:journalctl@ceph.osd.0.smithi022.stdout:Mar 21 12:51:50 smithi022 podman[112987]: 2024-03-21 12:51:50.603566355 +0000 UTC m=+0.709323786 container died 8a60e94f91184006fde9ff85f55df701ba1c48f4f52c1f895f30fbb37933fed7 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-0-deactivate, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.build-date=20240319, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-03-21T12:51:51.255 DEBUG:teuthology.orchestra.run.smithi022:> sudo pkill -f 'journalctl -f -n 0 -u ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715@osd.0.service' 2024-03-21T12:51:51.296 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-21T12:51:51.296 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-03-21T12:51:51.296 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-03-21T12:51:51.296 DEBUG:teuthology.orchestra.run.smithi022:> sudo systemctl stop ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715@osd.1 2024-03-21T12:51:51.594 INFO:journalctl@ceph.osd.1.smithi022.stdout:Mar 21 12:51:51 smithi022 systemd[1]: Stopping Ceph osd.1 for 3fc7f0f2-e77e-11ee-95cd-87774f69a715... 2024-03-21T12:51:52.094 INFO:journalctl@ceph.osd.1.smithi022.stdout:Mar 21 12:51:51 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-1[75646]: 2024-03-21T12:51:51.651+0000 7f86c3fb4700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-21T12:51:52.094 INFO:journalctl@ceph.osd.1.smithi022.stdout:Mar 21 12:51:51 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-1[75646]: 2024-03-21T12:51:51.651+0000 7f86c3fb4700 -1 osd.1 164 *** Got signal Terminated *** 2024-03-21T12:51:52.094 INFO:journalctl@ceph.osd.1.smithi022.stdout:Mar 21 12:51:51 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-1[75646]: 2024-03-21T12:51:51.651+0000 7f86c3fb4700 -1 osd.1 164 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-21T12:51:57.445 INFO:journalctl@ceph.osd.1.smithi022.stdout:Mar 21 12:51:57 smithi022 podman[113091]: 2024-03-21 12:51:57.134235966 +0000 UTC m=+5.599820283 container died 0f90e70bbf0a14657e3b081b7429efc76d72f983daf098524e30a26f32010134 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-1, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, org.label-schema.build-date=20240319, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.33.5) 2024-03-21T12:51:57.769 INFO:journalctl@ceph.osd.1.smithi022.stdout:Mar 21 12:51:57 smithi022 podman[113091]: 2024-03-21 12:51:57.52741816 +0000 UTC m=+5.993002482 container cleanup 0f90e70bbf0a14657e3b081b7429efc76d72f983daf098524e30a26f32010134 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-1, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.build-date=20240319, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.schema-version=1.0, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-03-21T12:51:57.770 INFO:journalctl@ceph.osd.1.smithi022.stdout:Mar 21 12:51:57 smithi022 bash[113091]: ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-1 2024-03-21T12:51:58.094 INFO:journalctl@ceph.osd.1.smithi022.stdout:Mar 21 12:51:57 smithi022 podman[113110]: 2024-03-21 12:51:57.770406096 +0000 UTC m=+0.639575169 container remove 0f90e70bbf0a14657e3b081b7429efc76d72f983daf098524e30a26f32010134 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-1, io.buildah.version=1.33.5, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240319, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, CEPH_POINT_RELEASE=, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, org.label-schema.vendor=CentOS) 2024-03-21T12:51:58.373 INFO:journalctl@ceph.osd.1.smithi022.stdout:Mar 21 12:51:58 smithi022 podman[113200]: 2024-03-21 12:51:58.014378695 +0000 UTC m=+0.022960898 image pull 19e12293512ff7978c8adb5f72a829060dd47f07bfd6903b6b3178d2ebc29e4e quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93 2024-03-21T12:51:58.373 INFO:journalctl@ceph.osd.1.smithi022.stdout:Mar 21 12:51:58 smithi022 podman[113200]: 2024-03-21 12:51:58.138122368 +0000 UTC m=+0.146704567 container create f48bb2ba9a850302903f70c6946f364c5595d951d9a8ea9b80b168d35d17fb3d (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-1-deactivate, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, org.label-schema.build-date=20240319, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-21T12:51:58.373 INFO:journalctl@ceph.osd.1.smithi022.stdout:Mar 21 12:51:58 smithi022 podman[113200]: 2024-03-21 12:51:58.373895092 +0000 UTC m=+0.382477292 container init f48bb2ba9a850302903f70c6946f364c5595d951d9a8ea9b80b168d35d17fb3d (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-1-deactivate, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_BRANCH=HEAD, org.label-schema.build-date=20240319, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=) 2024-03-21T12:51:58.670 INFO:journalctl@ceph.osd.1.smithi022.stdout:Mar 21 12:51:58 smithi022 podman[113200]: 2024-03-21 12:51:58.377145557 +0000 UTC m=+0.385727747 container start f48bb2ba9a850302903f70c6946f364c5595d951d9a8ea9b80b168d35d17fb3d (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-1-deactivate, org.label-schema.license=GPLv2, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.33.5, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240319, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True) 2024-03-21T12:51:58.670 INFO:journalctl@ceph.osd.1.smithi022.stdout:Mar 21 12:51:58 smithi022 podman[113200]: 2024-03-21 12:51:58.482386732 +0000 UTC m=+0.490968922 container attach f48bb2ba9a850302903f70c6946f364c5595d951d9a8ea9b80b168d35d17fb3d (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-1-deactivate, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, org.label-schema.build-date=20240319, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-03-21T12:51:59.036 INFO:journalctl@ceph.osd.1.smithi022.stdout:Mar 21 12:51:58 smithi022 podman[113200]: 2024-03-21 12:51:58.67055918 +0000 UTC m=+0.679141382 container died f48bb2ba9a850302903f70c6946f364c5595d951d9a8ea9b80b168d35d17fb3d (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-1-deactivate, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240319, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe) 2024-03-21T12:51:59.338 DEBUG:teuthology.orchestra.run.smithi022:> sudo pkill -f 'journalctl -f -n 0 -u ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715@osd.1.service' 2024-03-21T12:51:59.381 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-21T12:51:59.381 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-03-21T12:51:59.382 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-03-21T12:51:59.382 DEBUG:teuthology.orchestra.run.smithi022:> sudo systemctl stop ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715@osd.2 2024-03-21T12:51:59.594 INFO:journalctl@ceph.osd.2.smithi022.stdout:Mar 21 12:51:59 smithi022 systemd[1]: Stopping Ceph osd.2 for 3fc7f0f2-e77e-11ee-95cd-87774f69a715... 2024-03-21T12:52:00.094 INFO:journalctl@ceph.osd.2.smithi022.stdout:Mar 21 12:51:59 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-2[78185]: 2024-03-21T12:51:59.730+0000 7fe33571d700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-21T12:52:00.094 INFO:journalctl@ceph.osd.2.smithi022.stdout:Mar 21 12:51:59 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-2[78185]: 2024-03-21T12:51:59.730+0000 7fe33571d700 -1 osd.2 164 *** Got signal Terminated *** 2024-03-21T12:52:00.094 INFO:journalctl@ceph.osd.2.smithi022.stdout:Mar 21 12:51:59 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-2[78185]: 2024-03-21T12:51:59.730+0000 7fe33571d700 -1 osd.2 164 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-21T12:52:05.541 INFO:journalctl@ceph.osd.2.smithi022.stdout:Mar 21 12:52:05 smithi022 podman[113305]: 2024-03-21 12:52:05.254509868 +0000 UTC m=+5.647494042 container died 78a229f593f848a2c11074bac4d2ef708806f2c16fda03dd345892e5718d3395 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.build-date=20240319, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-03-21T12:52:05.844 INFO:journalctl@ceph.osd.2.smithi022.stdout:Mar 21 12:52:05 smithi022 podman[113305]: 2024-03-21 12:52:05.622314217 +0000 UTC m=+6.015298381 container cleanup 78a229f593f848a2c11074bac4d2ef708806f2c16fda03dd345892e5718d3395 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-2, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, CEPH_POINT_RELEASE=, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.build-date=20240319, GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-03-21T12:52:05.845 INFO:journalctl@ceph.osd.2.smithi022.stdout:Mar 21 12:52:05 smithi022 bash[113305]: ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-2 2024-03-21T12:52:06.244 INFO:journalctl@ceph.osd.2.smithi022.stdout:Mar 21 12:52:05 smithi022 podman[113354]: 2024-03-21 12:52:05.907134504 +0000 UTC m=+0.653182599 container remove 78a229f593f848a2c11074bac4d2ef708806f2c16fda03dd345892e5718d3395 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240319, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30) 2024-03-21T12:52:06.552 INFO:journalctl@ceph.osd.2.smithi022.stdout:Mar 21 12:52:06 smithi022 podman[113447]: 2024-03-21 12:52:06.148824444 +0000 UTC m=+0.018632290 image pull 19e12293512ff7978c8adb5f72a829060dd47f07bfd6903b6b3178d2ebc29e4e quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93 2024-03-21T12:52:06.552 INFO:journalctl@ceph.osd.2.smithi022.stdout:Mar 21 12:52:06 smithi022 podman[113447]: 2024-03-21 12:52:06.26659808 +0000 UTC m=+0.136405925 container create d1151e89bc81dcf9324be43d6a2b08268f779ca31659ef4b0aa62c186ddd5025 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-2-deactivate, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, org.label-schema.license=GPLv2, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, ceph=True, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240319, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-03-21T12:52:06.552 INFO:journalctl@ceph.osd.2.smithi022.stdout:Mar 21 12:52:06 smithi022 podman[113447]: 2024-03-21 12:52:06.477302934 +0000 UTC m=+0.347110792 container init d1151e89bc81dcf9324be43d6a2b08268f779ca31659ef4b0aa62c186ddd5025 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-2-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.build-date=20240319, GIT_CLEAN=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True) 2024-03-21T12:52:06.552 INFO:journalctl@ceph.osd.2.smithi022.stdout:Mar 21 12:52:06 smithi022 podman[113447]: 2024-03-21 12:52:06.480727515 +0000 UTC m=+0.350535361 container start d1151e89bc81dcf9324be43d6a2b08268f779ca31659ef4b0aa62c186ddd5025 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-2-deactivate, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240319, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-03-21T12:52:06.552 INFO:journalctl@ceph.osd.2.smithi022.stdout:Mar 21 12:52:06 smithi022 podman[113447]: 2024-03-21 12:52:06.552377976 +0000 UTC m=+0.422185845 container attach d1151e89bc81dcf9324be43d6a2b08268f779ca31659ef4b0aa62c186ddd5025 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-2-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.33.5, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240319, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-03-21T12:52:06.844 INFO:journalctl@ceph.osd.2.smithi022.stdout:Mar 21 12:52:06 smithi022 podman[113447]: 2024-03-21 12:52:06.779698073 +0000 UTC m=+0.649505921 container died d1151e89bc81dcf9324be43d6a2b08268f779ca31659ef4b0aa62c186ddd5025 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-2-deactivate, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, CEPH_POINT_RELEASE=, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, ceph=True, org.label-schema.build-date=20240319) 2024-03-21T12:52:07.359 DEBUG:teuthology.orchestra.run.smithi022:> sudo pkill -f 'journalctl -f -n 0 -u ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715@osd.2.service' 2024-03-21T12:52:07.400 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-21T12:52:07.400 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-03-21T12:52:07.400 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-03-21T12:52:07.400 DEBUG:teuthology.orchestra.run.smithi022:> sudo systemctl stop ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715@osd.3 2024-03-21T12:52:07.594 INFO:journalctl@ceph.osd.3.smithi022.stdout:Mar 21 12:52:07 smithi022 systemd[1]: Stopping Ceph osd.3 for 3fc7f0f2-e77e-11ee-95cd-87774f69a715... 2024-03-21T12:52:08.095 INFO:journalctl@ceph.osd.3.smithi022.stdout:Mar 21 12:52:07 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-3[80861]: 2024-03-21T12:52:07.732+0000 7fc2b83e3700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-21T12:52:08.095 INFO:journalctl@ceph.osd.3.smithi022.stdout:Mar 21 12:52:07 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-3[80861]: 2024-03-21T12:52:07.733+0000 7fc2b83e3700 -1 osd.3 164 *** Got signal Terminated *** 2024-03-21T12:52:08.095 INFO:journalctl@ceph.osd.3.smithi022.stdout:Mar 21 12:52:07 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-3[80861]: 2024-03-21T12:52:07.733+0000 7fc2b83e3700 -1 osd.3 164 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-21T12:52:10.344 INFO:journalctl@ceph.osd.3.smithi022.stdout:Mar 21 12:52:10 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-3[80861]: 2024-03-21T12:52:10.034+0000 7fc2b49d7700 -1 osd.3 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:46.731660+0000 front 2024-03-21T12:51:46.731675+0000 (oldest deadline 2024-03-21T12:52:09.631670+0000) 2024-03-21T12:52:10.786 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:52:10 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-5[60346]: 2024-03-21T12:52:10.345+0000 7f06268d6700 -1 osd.5 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:46.649108+0000 front 2024-03-21T12:51:46.649056+0000 (oldest deadline 2024-03-21T12:52:10.149001+0000) 2024-03-21T12:52:11.344 INFO:journalctl@ceph.osd.3.smithi022.stdout:Mar 21 12:52:11 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-3[80861]: 2024-03-21T12:52:11.017+0000 7fc2b49d7700 -1 osd.3 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:46.731660+0000 front 2024-03-21T12:51:46.731675+0000 (oldest deadline 2024-03-21T12:52:09.631670+0000) 2024-03-21T12:52:11.535 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:52:11 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-5[60346]: 2024-03-21T12:52:11.309+0000 7f06268d6700 -1 osd.5 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:46.649108+0000 front 2024-03-21T12:51:46.649056+0000 (oldest deadline 2024-03-21T12:52:10.149001+0000) 2024-03-21T12:52:12.036 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:11 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:11.730+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:43.900855+0000 front 2024-03-21T12:51:43.900987+0000 (oldest deadline 2024-03-21T12:52:11.501313+0000) 2024-03-21T12:52:12.344 INFO:journalctl@ceph.osd.3.smithi022.stdout:Mar 21 12:52:12 smithi022 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-3[80861]: 2024-03-21T12:52:12.035+0000 7fc2b49d7700 -1 osd.3 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:46.731660+0000 front 2024-03-21T12:51:46.731675+0000 (oldest deadline 2024-03-21T12:52:09.631670+0000) 2024-03-21T12:52:12.536 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:52:12 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-5[60346]: 2024-03-21T12:52:12.262+0000 7f06268d6700 -1 osd.5 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:46.649108+0000 front 2024-03-21T12:51:46.649056+0000 (oldest deadline 2024-03-21T12:52:10.149001+0000) 2024-03-21T12:52:13.035 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:12 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:12.687+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:43.900855+0000 front 2024-03-21T12:51:43.900987+0000 (oldest deadline 2024-03-21T12:52:11.501313+0000) 2024-03-21T12:52:13.537 INFO:journalctl@ceph.osd.3.smithi022.stdout:Mar 21 12:52:13 smithi022 podman[113553]: 2024-03-21 12:52:13.218014543 +0000 UTC m=+5.588175549 container died 61a63b67d9aff27721d31354e6c7933708b0365d59e532f2269070bd8a8c1412 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-3, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, org.label-schema.build-date=20240319, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS) 2024-03-21T12:52:13.699 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:52:13 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-5[60346]: 2024-03-21T12:52:13.289+0000 7f06268d6700 -1 osd.5 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:46.649108+0000 front 2024-03-21T12:51:46.649056+0000 (oldest deadline 2024-03-21T12:52:10.149001+0000) 2024-03-21T12:52:13.699 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 21 12:52:13 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-4[58075]: 2024-03-21T12:52:13.326+0000 7f88cdc04700 -1 osd.4 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:46.302077+0000 front 2024-03-21T12:51:46.302040+0000 (oldest deadline 2024-03-21T12:52:13.302748+0000) 2024-03-21T12:52:13.844 INFO:journalctl@ceph.osd.3.smithi022.stdout:Mar 21 12:52:13 smithi022 podman[113553]: 2024-03-21 12:52:13.70004142 +0000 UTC m=+6.070202424 container cleanup 61a63b67d9aff27721d31354e6c7933708b0365d59e532f2269070bd8a8c1412 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-3, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, org.label-schema.build-date=20240319, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-03-21T12:52:13.844 INFO:journalctl@ceph.osd.3.smithi022.stdout:Mar 21 12:52:13 smithi022 bash[113553]: ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-3 2024-03-21T12:52:14.036 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:13 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:13.698+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:43.900855+0000 front 2024-03-21T12:51:43.900987+0000 (oldest deadline 2024-03-21T12:52:11.501313+0000) 2024-03-21T12:52:14.332 INFO:journalctl@ceph.osd.3.smithi022.stdout:Mar 21 12:52:13 smithi022 podman[113571]: 2024-03-21 12:52:13.968444806 +0000 UTC m=+0.748756598 container remove 61a63b67d9aff27721d31354e6c7933708b0365d59e532f2269070bd8a8c1412 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-3, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240319, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux ) 2024-03-21T12:52:14.584 INFO:journalctl@ceph.osd.3.smithi022.stdout:Mar 21 12:52:14 smithi022 podman[113663]: 2024-03-21 12:52:14.234686844 +0000 UTC m=+0.017870815 image pull 19e12293512ff7978c8adb5f72a829060dd47f07bfd6903b6b3178d2ebc29e4e quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93 2024-03-21T12:52:14.584 INFO:journalctl@ceph.osd.3.smithi022.stdout:Mar 21 12:52:14 smithi022 podman[113663]: 2024-03-21 12:52:14.344243889 +0000 UTC m=+0.127427850 container create a340b620a74110178b25a2f8e55f0351282249acc170fe96d45c2c6c9a5f755e (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-3-deactivate, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240319, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe) 2024-03-21T12:52:14.584 INFO:journalctl@ceph.osd.3.smithi022.stdout:Mar 21 12:52:14 smithi022 podman[113663]: 2024-03-21 12:52:14.580373092 +0000 UTC m=+0.363557061 container init a340b620a74110178b25a2f8e55f0351282249acc170fe96d45c2c6c9a5f755e (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-3-deactivate, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240319, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-03-21T12:52:14.584 INFO:journalctl@ceph.osd.3.smithi022.stdout:Mar 21 12:52:14 smithi022 podman[113663]: 2024-03-21 12:52:14.584929265 +0000 UTC m=+0.368113227 container start a340b620a74110178b25a2f8e55f0351282249acc170fe96d45c2c6c9a5f755e (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-3-deactivate, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, org.label-schema.build-date=20240319, ceph=True, GIT_CLEAN=True, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-21T12:52:14.735 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 21 12:52:14 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-4[58075]: 2024-03-21T12:52:14.293+0000 7f88cdc04700 -1 osd.4 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:46.302077+0000 front 2024-03-21T12:51:46.302040+0000 (oldest deadline 2024-03-21T12:52:13.302748+0000) 2024-03-21T12:52:14.736 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:52:14 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-5[60346]: 2024-03-21T12:52:14.332+0000 7f06268d6700 -1 osd.5 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:46.649108+0000 front 2024-03-21T12:51:46.649056+0000 (oldest deadline 2024-03-21T12:52:10.149001+0000) 2024-03-21T12:52:14.736 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:52:14 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6[62661]: 2024-03-21T12:52:14.502+0000 7fbdb0c72700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:46.295470+0000 front 2024-03-21T12:51:46.295565+0000 (oldest deadline 2024-03-21T12:52:14.496180+0000) 2024-03-21T12:52:14.844 INFO:journalctl@ceph.osd.3.smithi022.stdout:Mar 21 12:52:14 smithi022 podman[113663]: 2024-03-21 12:52:14.680140482 +0000 UTC m=+0.463324459 container attach a340b620a74110178b25a2f8e55f0351282249acc170fe96d45c2c6c9a5f755e (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-3-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, org.label-schema.build-date=20240319, maintainer=Guillaume Abrioux , GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-21T12:52:15.036 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:14 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:14.735+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:43.900855+0000 front 2024-03-21T12:51:43.900987+0000 (oldest deadline 2024-03-21T12:52:11.501313+0000) 2024-03-21T12:52:15.344 INFO:journalctl@ceph.osd.3.smithi022.stdout:Mar 21 12:52:14 smithi022 podman[113663]: 2024-03-21 12:52:14.895735977 +0000 UTC m=+0.678919947 container died a340b620a74110178b25a2f8e55f0351282249acc170fe96d45c2c6c9a5f755e (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-3-deactivate, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240319, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe) 2024-03-21T12:52:15.480 DEBUG:teuthology.orchestra.run.smithi022:> sudo pkill -f 'journalctl -f -n 0 -u ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715@osd.3.service' 2024-03-21T12:52:15.520 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-21T12:52:15.520 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-03-21T12:52:15.520 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-03-21T12:52:15.520 DEBUG:teuthology.orchestra.run.smithi189:> sudo systemctl stop ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715@osd.4 2024-03-21T12:52:15.553 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 21 12:52:15 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-4[58075]: 2024-03-21T12:52:15.289+0000 7f88cdc04700 -1 osd.4 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:46.302077+0000 front 2024-03-21T12:51:46.302040+0000 (oldest deadline 2024-03-21T12:52:13.302748+0000) 2024-03-21T12:52:15.554 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:52:15 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-5[60346]: 2024-03-21T12:52:15.337+0000 7f06268d6700 -1 osd.5 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:46.649108+0000 front 2024-03-21T12:51:46.649056+0000 (oldest deadline 2024-03-21T12:52:10.149001+0000) 2024-03-21T12:52:15.554 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:52:15 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6[62661]: 2024-03-21T12:52:15.457+0000 7fbdb0c72700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:46.295470+0000 front 2024-03-21T12:51:46.295565+0000 (oldest deadline 2024-03-21T12:52:14.496180+0000) 2024-03-21T12:52:15.842 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 21 12:52:15 smithi189 systemd[1]: Stopping Ceph osd.4 for 3fc7f0f2-e77e-11ee-95cd-87774f69a715... 2024-03-21T12:52:15.842 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:15 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:15.734+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:43.900855+0000 front 2024-03-21T12:51:43.900987+0000 (oldest deadline 2024-03-21T12:52:11.501313+0000) 2024-03-21T12:52:16.285 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 21 12:52:15 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-4[58075]: 2024-03-21T12:52:15.841+0000 7f88d1610700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-21T12:52:16.286 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 21 12:52:15 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-4[58075]: 2024-03-21T12:52:15.841+0000 7f88d1610700 -1 osd.4 164 *** Got signal Terminated *** 2024-03-21T12:52:16.286 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 21 12:52:15 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-4[58075]: 2024-03-21T12:52:15.841+0000 7f88d1610700 -1 osd.4 164 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-21T12:52:16.730 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 21 12:52:16 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-4[58075]: 2024-03-21T12:52:16.336+0000 7f88cdc04700 -1 osd.4 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:46.302077+0000 front 2024-03-21T12:51:46.302040+0000 (oldest deadline 2024-03-21T12:52:13.302748+0000) 2024-03-21T12:52:16.730 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:52:16 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-5[60346]: 2024-03-21T12:52:16.330+0000 7f06268d6700 -1 osd.5 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:46.649108+0000 front 2024-03-21T12:51:46.649056+0000 (oldest deadline 2024-03-21T12:52:10.149001+0000) 2024-03-21T12:52:16.730 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:52:16 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6[62661]: 2024-03-21T12:52:16.440+0000 7fbdb0c72700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:46.295470+0000 front 2024-03-21T12:51:46.295565+0000 (oldest deadline 2024-03-21T12:52:14.496180+0000) 2024-03-21T12:52:17.036 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:16 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:16.729+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:43.900855+0000 front 2024-03-21T12:51:43.900987+0000 (oldest deadline 2024-03-21T12:52:11.501313+0000) 2024-03-21T12:52:17.536 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:52:17 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-5[60346]: 2024-03-21T12:52:17.282+0000 7f06268d6700 -1 osd.5 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:46.649108+0000 front 2024-03-21T12:51:46.649056+0000 (oldest deadline 2024-03-21T12:52:10.149001+0000) 2024-03-21T12:52:17.536 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 21 12:52:17 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-4[58075]: 2024-03-21T12:52:17.307+0000 7f88cdc04700 -1 osd.4 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:46.302077+0000 front 2024-03-21T12:51:46.302040+0000 (oldest deadline 2024-03-21T12:52:13.302748+0000) 2024-03-21T12:52:17.536 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:52:17 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6[62661]: 2024-03-21T12:52:17.390+0000 7fbdb0c72700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:46.295470+0000 front 2024-03-21T12:51:46.295565+0000 (oldest deadline 2024-03-21T12:52:14.496180+0000) 2024-03-21T12:52:18.035 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:17 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:17.704+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:43.900855+0000 front 2024-03-21T12:51:43.900987+0000 (oldest deadline 2024-03-21T12:52:11.501313+0000) 2024-03-21T12:52:18.036 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:17 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:17.704+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-03-21T12:51:51.501922+0000 front 2024-03-21T12:51:51.501788+0000 (oldest deadline 2024-03-21T12:52:17.401661+0000) 2024-03-21T12:52:18.712 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 21 12:52:18 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-4[58075]: 2024-03-21T12:52:18.328+0000 7f88cdc04700 -1 osd.4 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:46.302077+0000 front 2024-03-21T12:51:46.302040+0000 (oldest deadline 2024-03-21T12:52:13.302748+0000) 2024-03-21T12:52:18.712 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:52:18 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-5[60346]: 2024-03-21T12:52:18.324+0000 7f06268d6700 -1 osd.5 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:46.649108+0000 front 2024-03-21T12:51:46.649056+0000 (oldest deadline 2024-03-21T12:52:10.149001+0000) 2024-03-21T12:52:18.713 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:52:18 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6[62661]: 2024-03-21T12:52:18.431+0000 7fbdb0c72700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:46.295470+0000 front 2024-03-21T12:51:46.295565+0000 (oldest deadline 2024-03-21T12:52:14.496180+0000) 2024-03-21T12:52:19.036 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:18 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:18.712+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:43.900855+0000 front 2024-03-21T12:51:43.900987+0000 (oldest deadline 2024-03-21T12:52:11.501313+0000) 2024-03-21T12:52:19.036 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:18 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:18.712+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-03-21T12:51:51.501922+0000 front 2024-03-21T12:51:51.501788+0000 (oldest deadline 2024-03-21T12:52:17.401661+0000) 2024-03-21T12:52:19.688 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:52:19 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6[62661]: 2024-03-21T12:52:19.383+0000 7fbdb0c72700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:46.295470+0000 front 2024-03-21T12:51:46.295565+0000 (oldest deadline 2024-03-21T12:52:14.496180+0000) 2024-03-21T12:52:19.688 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 21 12:52:19 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-4[58075]: 2024-03-21T12:52:19.355+0000 7f88cdc04700 -1 osd.4 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:46.302077+0000 front 2024-03-21T12:51:46.302040+0000 (oldest deadline 2024-03-21T12:52:13.302748+0000) 2024-03-21T12:52:19.688 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 21 12:52:19 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-4[58075]: 2024-03-21T12:52:19.355+0000 7f88cdc04700 -1 osd.4 164 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-03-21T12:51:53.303293+0000 front 2024-03-21T12:51:53.303310+0000 (oldest deadline 2024-03-21T12:52:19.104100+0000) 2024-03-21T12:52:19.688 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:52:19 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-5[60346]: 2024-03-21T12:52:19.335+0000 7f06268d6700 -1 osd.5 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:46.649108+0000 front 2024-03-21T12:51:46.649056+0000 (oldest deadline 2024-03-21T12:52:10.149001+0000) 2024-03-21T12:52:19.689 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:52:19 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-5[60346]: 2024-03-21T12:52:19.335+0000 7f06268d6700 -1 osd.5 164 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-03-21T12:51:53.049900+0000 front 2024-03-21T12:51:53.049805+0000 (oldest deadline 2024-03-21T12:52:18.949781+0000) 2024-03-21T12:52:20.035 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:19 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:19.687+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:43.900855+0000 front 2024-03-21T12:51:43.900987+0000 (oldest deadline 2024-03-21T12:52:11.501313+0000) 2024-03-21T12:52:20.036 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:19 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:19.687+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-03-21T12:51:51.501922+0000 front 2024-03-21T12:51:51.501788+0000 (oldest deadline 2024-03-21T12:52:17.401661+0000) 2024-03-21T12:52:20.729 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:52:20 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6[62661]: 2024-03-21T12:52:20.377+0000 7fbdb0c72700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:46.295470+0000 front 2024-03-21T12:51:46.295565+0000 (oldest deadline 2024-03-21T12:52:14.496180+0000) 2024-03-21T12:52:20.729 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:52:20 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6[62661]: 2024-03-21T12:52:20.377+0000 7fbdb0c72700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-03-21T12:51:55.597270+0000 front 2024-03-21T12:51:55.597221+0000 (oldest deadline 2024-03-21T12:52:20.297338+0000) 2024-03-21T12:52:20.729 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 21 12:52:20 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-4[58075]: 2024-03-21T12:52:20.338+0000 7f88cdc04700 -1 osd.4 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:46.302077+0000 front 2024-03-21T12:51:46.302040+0000 (oldest deadline 2024-03-21T12:52:13.302748+0000) 2024-03-21T12:52:20.729 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 21 12:52:20 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-4[58075]: 2024-03-21T12:52:20.338+0000 7f88cdc04700 -1 osd.4 164 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-03-21T12:51:53.303293+0000 front 2024-03-21T12:51:53.303310+0000 (oldest deadline 2024-03-21T12:52:19.104100+0000) 2024-03-21T12:52:20.729 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:52:20 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-5[60346]: 2024-03-21T12:52:20.352+0000 7f06268d6700 -1 osd.5 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:46.649108+0000 front 2024-03-21T12:51:46.649056+0000 (oldest deadline 2024-03-21T12:52:10.149001+0000) 2024-03-21T12:52:20.729 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:52:20 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-5[60346]: 2024-03-21T12:52:20.352+0000 7f06268d6700 -1 osd.5 164 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-03-21T12:51:53.049900+0000 front 2024-03-21T12:51:53.049805+0000 (oldest deadline 2024-03-21T12:52:18.949781+0000) 2024-03-21T12:52:21.036 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:20 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:20.728+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:43.900855+0000 front 2024-03-21T12:51:43.900987+0000 (oldest deadline 2024-03-21T12:52:11.501313+0000) 2024-03-21T12:52:21.036 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:20 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:20.728+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-03-21T12:51:51.501922+0000 front 2024-03-21T12:51:51.501788+0000 (oldest deadline 2024-03-21T12:52:17.401661+0000) 2024-03-21T12:52:21.692 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 21 12:52:21 smithi189 podman[79419]: 2024-03-21 12:52:21.366049226 +0000 UTC m=+5.644827555 container died 382755cee998bbface39691dbccd0c2f5550e133b6e9d4e9768f2935ef512d1c (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-4, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240319, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.vendor=CentOS, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, ceph=True) 2024-03-21T12:52:21.693 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:52:21 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-5[60346]: 2024-03-21T12:52:21.367+0000 7f06268d6700 -1 osd.5 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:46.649108+0000 front 2024-03-21T12:51:46.649056+0000 (oldest deadline 2024-03-21T12:52:10.149001+0000) 2024-03-21T12:52:21.693 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:52:21 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-5[60346]: 2024-03-21T12:52:21.367+0000 7f06268d6700 -1 osd.5 164 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-03-21T12:51:53.049900+0000 front 2024-03-21T12:51:53.049805+0000 (oldest deadline 2024-03-21T12:52:18.949781+0000) 2024-03-21T12:52:21.694 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:52:21 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6[62661]: 2024-03-21T12:52:21.376+0000 7fbdb0c72700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:46.295470+0000 front 2024-03-21T12:51:46.295565+0000 (oldest deadline 2024-03-21T12:52:14.496180+0000) 2024-03-21T12:52:21.694 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:52:21 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6[62661]: 2024-03-21T12:52:21.376+0000 7fbdb0c72700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-03-21T12:51:55.597270+0000 front 2024-03-21T12:51:55.597221+0000 (oldest deadline 2024-03-21T12:52:20.297338+0000) 2024-03-21T12:52:22.008 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:21 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:21.736+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:43.900855+0000 front 2024-03-21T12:51:43.900987+0000 (oldest deadline 2024-03-21T12:52:11.501313+0000) 2024-03-21T12:52:22.008 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:21 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:21.736+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-03-21T12:51:51.501922+0000 front 2024-03-21T12:51:51.501788+0000 (oldest deadline 2024-03-21T12:52:17.401661+0000) 2024-03-21T12:52:22.008 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 21 12:52:21 smithi189 podman[79419]: 2024-03-21 12:52:21.892196561 +0000 UTC m=+6.170974884 container cleanup 382755cee998bbface39691dbccd0c2f5550e133b6e9d4e9768f2935ef512d1c (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-4, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240319, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True) 2024-03-21T12:52:22.008 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 21 12:52:21 smithi189 bash[79419]: ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-4 2024-03-21T12:52:22.286 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 21 12:52:22 smithi189 podman[79468]: 2024-03-21 12:52:22.118982122 +0000 UTC m=+0.752366457 container remove 382755cee998bbface39691dbccd0c2f5550e133b6e9d4e9768f2935ef512d1c (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-4, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.build-date=20240319, CEPH_POINT_RELEASE=) 2024-03-21T12:52:22.655 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:52:22 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-5[60346]: 2024-03-21T12:52:22.365+0000 7f06268d6700 -1 osd.5 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:46.649108+0000 front 2024-03-21T12:51:46.649056+0000 (oldest deadline 2024-03-21T12:52:10.149001+0000) 2024-03-21T12:52:22.655 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:52:22 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-5[60346]: 2024-03-21T12:52:22.365+0000 7f06268d6700 -1 osd.5 164 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-03-21T12:51:53.049900+0000 front 2024-03-21T12:51:53.049805+0000 (oldest deadline 2024-03-21T12:52:18.949781+0000) 2024-03-21T12:52:22.655 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:52:22 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6[62661]: 2024-03-21T12:52:22.348+0000 7fbdb0c72700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:46.295470+0000 front 2024-03-21T12:51:46.295565+0000 (oldest deadline 2024-03-21T12:52:14.496180+0000) 2024-03-21T12:52:22.655 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:52:22 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6[62661]: 2024-03-21T12:52:22.348+0000 7fbdb0c72700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-03-21T12:51:55.597270+0000 front 2024-03-21T12:51:55.597221+0000 (oldest deadline 2024-03-21T12:52:20.297338+0000) 2024-03-21T12:52:22.655 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 21 12:52:22 smithi189 podman[79566]: 2024-03-21 12:52:22.453457819 +0000 UTC m=+0.018065885 image pull 19e12293512ff7978c8adb5f72a829060dd47f07bfd6903b6b3178d2ebc29e4e quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93 2024-03-21T12:52:22.656 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 21 12:52:22 smithi189 podman[79566]: 2024-03-21 12:52:22.553016334 +0000 UTC m=+0.117624396 container create 9b58e95ca5988c1b4cf6a363ce9efd3caf36e5052664e372d952ed793e34eae5 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-4-deactivate, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, org.label-schema.build-date=20240319, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_BRANCH=HEAD) 2024-03-21T12:52:23.038 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:22 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:22.741+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:43.900855+0000 front 2024-03-21T12:51:43.900987+0000 (oldest deadline 2024-03-21T12:52:11.501313+0000) 2024-03-21T12:52:23.038 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:22 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:22.741+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-03-21T12:51:51.501922+0000 front 2024-03-21T12:51:51.501788+0000 (oldest deadline 2024-03-21T12:52:17.401661+0000) 2024-03-21T12:52:23.038 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 21 12:52:22 smithi189 podman[79566]: 2024-03-21 12:52:22.755478602 +0000 UTC m=+0.320086666 container init 9b58e95ca5988c1b4cf6a363ce9efd3caf36e5052664e372d952ed793e34eae5 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-4-deactivate, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.build-date=20240319, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-03-21T12:52:23.038 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 21 12:52:22 smithi189 podman[79566]: 2024-03-21 12:52:22.758545329 +0000 UTC m=+0.323153391 container start 9b58e95ca5988c1b4cf6a363ce9efd3caf36e5052664e372d952ed793e34eae5 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-4-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240319, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.license=GPLv2) 2024-03-21T12:52:23.038 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 21 12:52:22 smithi189 podman[79566]: 2024-03-21 12:52:22.855570191 +0000 UTC m=+0.420178265 container attach 9b58e95ca5988c1b4cf6a363ce9efd3caf36e5052664e372d952ed793e34eae5 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-4-deactivate, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240319, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-03-21T12:52:23.315 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 21 12:52:23 smithi189 podman[79566]: 2024-03-21 12:52:23.054070359 +0000 UTC m=+0.618678427 container died 9b58e95ca5988c1b4cf6a363ce9efd3caf36e5052664e372d952ed793e34eae5 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-4-deactivate, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240319) 2024-03-21T12:52:23.668 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:52:23 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-5[60346]: 2024-03-21T12:52:23.373+0000 7f06268d6700 -1 osd.5 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:46.649108+0000 front 2024-03-21T12:51:46.649056+0000 (oldest deadline 2024-03-21T12:52:10.149001+0000) 2024-03-21T12:52:23.668 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:52:23 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-5[60346]: 2024-03-21T12:52:23.373+0000 7f06268d6700 -1 osd.5 164 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-03-21T12:51:53.049900+0000 front 2024-03-21T12:51:53.049805+0000 (oldest deadline 2024-03-21T12:52:18.949781+0000) 2024-03-21T12:52:23.668 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:52:23 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6[62661]: 2024-03-21T12:52:23.314+0000 7fbdb0c72700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:46.295470+0000 front 2024-03-21T12:51:46.295565+0000 (oldest deadline 2024-03-21T12:52:14.496180+0000) 2024-03-21T12:52:23.668 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:52:23 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6[62661]: 2024-03-21T12:52:23.314+0000 7fbdb0c72700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-03-21T12:51:55.597270+0000 front 2024-03-21T12:51:55.597221+0000 (oldest deadline 2024-03-21T12:52:20.297338+0000) 2024-03-21T12:52:23.687 DEBUG:teuthology.orchestra.run.smithi189:> sudo pkill -f 'journalctl -f -n 0 -u ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715@osd.4.service' 2024-03-21T12:52:23.769 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-21T12:52:23.769 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-03-21T12:52:23.769 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-03-21T12:52:23.769 DEBUG:teuthology.orchestra.run.smithi189:> sudo systemctl stop ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715@osd.5 2024-03-21T12:52:24.035 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:52:23 smithi189 systemd[1]: Stopping Ceph osd.5 for 3fc7f0f2-e77e-11ee-95cd-87774f69a715... 2024-03-21T12:52:24.036 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:23 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:23.757+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:43.900855+0000 front 2024-03-21T12:51:43.900987+0000 (oldest deadline 2024-03-21T12:52:11.501313+0000) 2024-03-21T12:52:24.036 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:23 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:23.757+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-03-21T12:51:51.501922+0000 front 2024-03-21T12:51:51.501788+0000 (oldest deadline 2024-03-21T12:52:17.401661+0000) 2024-03-21T12:52:24.536 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:52:24 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6[62661]: 2024-03-21T12:52:24.354+0000 7fbdb0c72700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:46.295470+0000 front 2024-03-21T12:51:46.295565+0000 (oldest deadline 2024-03-21T12:52:14.496180+0000) 2024-03-21T12:52:24.536 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:52:24 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6[62661]: 2024-03-21T12:52:24.354+0000 7fbdb0c72700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-03-21T12:51:55.597270+0000 front 2024-03-21T12:51:55.597221+0000 (oldest deadline 2024-03-21T12:52:20.297338+0000) 2024-03-21T12:52:24.536 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:52:24 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-5[60346]: 2024-03-21T12:52:24.145+0000 7f062a2e2700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-21T12:52:24.536 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:52:24 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-5[60346]: 2024-03-21T12:52:24.145+0000 7f062a2e2700 -1 osd.5 164 *** Got signal Terminated *** 2024-03-21T12:52:24.536 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:52:24 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-5[60346]: 2024-03-21T12:52:24.145+0000 7f062a2e2700 -1 osd.5 164 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-21T12:52:24.536 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:52:24 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-5[60346]: 2024-03-21T12:52:24.385+0000 7f06268d6700 -1 osd.5 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:46.649108+0000 front 2024-03-21T12:51:46.649056+0000 (oldest deadline 2024-03-21T12:52:10.149001+0000) 2024-03-21T12:52:24.537 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:52:24 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-5[60346]: 2024-03-21T12:52:24.385+0000 7f06268d6700 -1 osd.5 164 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-03-21T12:51:53.049900+0000 front 2024-03-21T12:51:53.049805+0000 (oldest deadline 2024-03-21T12:52:18.949781+0000) 2024-03-21T12:52:25.035 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:24 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:24.772+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:43.900855+0000 front 2024-03-21T12:51:43.900987+0000 (oldest deadline 2024-03-21T12:52:11.501313+0000) 2024-03-21T12:52:25.035 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:24 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:24.772+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-03-21T12:51:51.501922+0000 front 2024-03-21T12:51:51.501788+0000 (oldest deadline 2024-03-21T12:52:17.401661+0000) 2024-03-21T12:52:25.785 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:52:25 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-5[60346]: 2024-03-21T12:52:25.366+0000 7f06268d6700 -1 osd.5 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:46.649108+0000 front 2024-03-21T12:51:46.649056+0000 (oldest deadline 2024-03-21T12:52:10.149001+0000) 2024-03-21T12:52:25.786 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:52:25 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-5[60346]: 2024-03-21T12:52:25.366+0000 7f06268d6700 -1 osd.5 164 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-03-21T12:51:53.049900+0000 front 2024-03-21T12:51:53.049805+0000 (oldest deadline 2024-03-21T12:52:18.949781+0000) 2024-03-21T12:52:25.786 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:52:25 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6[62661]: 2024-03-21T12:52:25.340+0000 7fbdb0c72700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:46.295470+0000 front 2024-03-21T12:51:46.295565+0000 (oldest deadline 2024-03-21T12:52:14.496180+0000) 2024-03-21T12:52:25.786 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:52:25 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6[62661]: 2024-03-21T12:52:25.340+0000 7fbdb0c72700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-03-21T12:51:55.597270+0000 front 2024-03-21T12:51:55.597221+0000 (oldest deadline 2024-03-21T12:52:20.297338+0000) 2024-03-21T12:52:26.285 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:25 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:25.799+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:43.900855+0000 front 2024-03-21T12:51:43.900987+0000 (oldest deadline 2024-03-21T12:52:11.501313+0000) 2024-03-21T12:52:26.286 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:25 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:25.799+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-03-21T12:51:51.501922+0000 front 2024-03-21T12:51:51.501788+0000 (oldest deadline 2024-03-21T12:52:17.401661+0000) 2024-03-21T12:52:26.786 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:52:26 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6[62661]: 2024-03-21T12:52:26.352+0000 7fbdb0c72700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:46.295470+0000 front 2024-03-21T12:51:46.295565+0000 (oldest deadline 2024-03-21T12:52:14.496180+0000) 2024-03-21T12:52:26.786 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:52:26 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6[62661]: 2024-03-21T12:52:26.352+0000 7fbdb0c72700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-03-21T12:51:55.597270+0000 front 2024-03-21T12:51:55.597221+0000 (oldest deadline 2024-03-21T12:52:20.297338+0000) 2024-03-21T12:52:26.786 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:52:26 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-5[60346]: 2024-03-21T12:52:26.365+0000 7f06268d6700 -1 osd.5 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:46.649108+0000 front 2024-03-21T12:51:46.649056+0000 (oldest deadline 2024-03-21T12:52:10.149001+0000) 2024-03-21T12:52:26.786 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:52:26 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-5[60346]: 2024-03-21T12:52:26.365+0000 7f06268d6700 -1 osd.5 164 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-03-21T12:51:53.049900+0000 front 2024-03-21T12:51:53.049805+0000 (oldest deadline 2024-03-21T12:52:18.949781+0000) 2024-03-21T12:52:26.786 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:52:26 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-5[60346]: 2024-03-21T12:52:26.365+0000 7f06268d6700 -1 osd.5 164 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-03-21T12:52:01.250432+0000 front 2024-03-21T12:52:01.250500+0000 (oldest deadline 2024-03-21T12:52:25.950510+0000) 2024-03-21T12:52:27.285 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:26 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:26.815+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:43.900855+0000 front 2024-03-21T12:51:43.900987+0000 (oldest deadline 2024-03-21T12:52:11.501313+0000) 2024-03-21T12:52:27.286 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:26 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:26.815+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-03-21T12:51:51.501922+0000 front 2024-03-21T12:51:51.501788+0000 (oldest deadline 2024-03-21T12:52:17.401661+0000) 2024-03-21T12:52:27.286 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:26 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:26.815+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-03-21T12:52:03.103687+0000 front 2024-03-21T12:52:03.103788+0000 (oldest deadline 2024-03-21T12:52:26.603833+0000) 2024-03-21T12:52:27.786 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:52:27 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6[62661]: 2024-03-21T12:52:27.379+0000 7fbdb0c72700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:46.295470+0000 front 2024-03-21T12:51:46.295565+0000 (oldest deadline 2024-03-21T12:52:14.496180+0000) 2024-03-21T12:52:27.786 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:52:27 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6[62661]: 2024-03-21T12:52:27.379+0000 7fbdb0c72700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-03-21T12:51:55.597270+0000 front 2024-03-21T12:51:55.597221+0000 (oldest deadline 2024-03-21T12:52:20.297338+0000) 2024-03-21T12:52:27.786 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:52:27 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-5[60346]: 2024-03-21T12:52:27.400+0000 7f06268d6700 -1 osd.5 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:46.649108+0000 front 2024-03-21T12:51:46.649056+0000 (oldest deadline 2024-03-21T12:52:10.149001+0000) 2024-03-21T12:52:27.786 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:52:27 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-5[60346]: 2024-03-21T12:52:27.400+0000 7f06268d6700 -1 osd.5 164 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-03-21T12:51:53.049900+0000 front 2024-03-21T12:51:53.049805+0000 (oldest deadline 2024-03-21T12:52:18.949781+0000) 2024-03-21T12:52:27.786 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:52:27 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-5[60346]: 2024-03-21T12:52:27.400+0000 7f06268d6700 -1 osd.5 164 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-03-21T12:52:01.250432+0000 front 2024-03-21T12:52:01.250500+0000 (oldest deadline 2024-03-21T12:52:25.950510+0000) 2024-03-21T12:52:28.286 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:27 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:27.856+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:43.900855+0000 front 2024-03-21T12:51:43.900987+0000 (oldest deadline 2024-03-21T12:52:11.501313+0000) 2024-03-21T12:52:28.286 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:27 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:27.856+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-03-21T12:51:51.501922+0000 front 2024-03-21T12:51:51.501788+0000 (oldest deadline 2024-03-21T12:52:17.401661+0000) 2024-03-21T12:52:28.286 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:27 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:27.856+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-03-21T12:52:03.103687+0000 front 2024-03-21T12:52:03.103788+0000 (oldest deadline 2024-03-21T12:52:26.603833+0000) 2024-03-21T12:52:28.786 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:52:28 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6[62661]: 2024-03-21T12:52:28.415+0000 7fbdb0c72700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:46.295470+0000 front 2024-03-21T12:51:46.295565+0000 (oldest deadline 2024-03-21T12:52:14.496180+0000) 2024-03-21T12:52:28.786 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:52:28 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6[62661]: 2024-03-21T12:52:28.415+0000 7fbdb0c72700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-03-21T12:51:55.597270+0000 front 2024-03-21T12:51:55.597221+0000 (oldest deadline 2024-03-21T12:52:20.297338+0000) 2024-03-21T12:52:28.786 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:52:28 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-5[60346]: 2024-03-21T12:52:28.381+0000 7f06268d6700 -1 osd.5 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:46.649108+0000 front 2024-03-21T12:51:46.649056+0000 (oldest deadline 2024-03-21T12:52:10.149001+0000) 2024-03-21T12:52:28.786 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:52:28 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-5[60346]: 2024-03-21T12:52:28.381+0000 7f06268d6700 -1 osd.5 164 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-03-21T12:51:53.049900+0000 front 2024-03-21T12:51:53.049805+0000 (oldest deadline 2024-03-21T12:52:18.949781+0000) 2024-03-21T12:52:28.789 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:52:28 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-5[60346]: 2024-03-21T12:52:28.381+0000 7f06268d6700 -1 osd.5 164 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-03-21T12:52:01.250432+0000 front 2024-03-21T12:52:01.250500+0000 (oldest deadline 2024-03-21T12:52:25.950510+0000) 2024-03-21T12:52:29.286 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:28 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:28.820+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:43.900855+0000 front 2024-03-21T12:51:43.900987+0000 (oldest deadline 2024-03-21T12:52:11.501313+0000) 2024-03-21T12:52:29.286 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:28 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:28.820+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-03-21T12:51:51.501922+0000 front 2024-03-21T12:51:51.501788+0000 (oldest deadline 2024-03-21T12:52:17.401661+0000) 2024-03-21T12:52:29.286 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:28 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:28.820+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-03-21T12:52:03.103687+0000 front 2024-03-21T12:52:03.103788+0000 (oldest deadline 2024-03-21T12:52:26.603833+0000) 2024-03-21T12:52:29.737 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:52:29 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6[62661]: 2024-03-21T12:52:29.449+0000 7fbdb0c72700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:46.295470+0000 front 2024-03-21T12:51:46.295565+0000 (oldest deadline 2024-03-21T12:52:14.496180+0000) 2024-03-21T12:52:29.737 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:52:29 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6[62661]: 2024-03-21T12:52:29.449+0000 7fbdb0c72700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-03-21T12:51:55.597270+0000 front 2024-03-21T12:51:55.597221+0000 (oldest deadline 2024-03-21T12:52:20.297338+0000) 2024-03-21T12:52:29.737 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:52:29 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6[62661]: 2024-03-21T12:52:29.449+0000 7fbdb0c72700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-03-21T12:52:04.398328+0000 front 2024-03-21T12:52:04.398220+0000 (oldest deadline 2024-03-21T12:52:28.498321+0000) 2024-03-21T12:52:30.012 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:52:29 smithi189 podman[79682]: 2024-03-21 12:52:29.737178355 +0000 UTC m=+5.718281676 container died d2c15ba54b4f9587306441a10cb65af7d311a50b1337c97c68b9f8e5b823b288 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-5, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.build-date=20240319, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, ceph=True, GIT_CLEAN=True, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-21T12:52:30.013 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:29 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:29.866+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:43.900855+0000 front 2024-03-21T12:51:43.900987+0000 (oldest deadline 2024-03-21T12:52:11.501313+0000) 2024-03-21T12:52:30.013 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:29 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:29.866+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-03-21T12:51:51.501922+0000 front 2024-03-21T12:51:51.501788+0000 (oldest deadline 2024-03-21T12:52:17.401661+0000) 2024-03-21T12:52:30.013 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:29 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:29.866+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-03-21T12:52:03.103687+0000 front 2024-03-21T12:52:03.103788+0000 (oldest deadline 2024-03-21T12:52:26.603833+0000) 2024-03-21T12:52:30.286 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:52:30 smithi189 podman[79682]: 2024-03-21 12:52:30.095593324 +0000 UTC m=+6.076696643 container cleanup d2c15ba54b4f9587306441a10cb65af7d311a50b1337c97c68b9f8e5b823b288 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-5, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.build-date=20240319, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0) 2024-03-21T12:52:30.286 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:52:30 smithi189 bash[79682]: ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-5 2024-03-21T12:52:30.709 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:52:30 smithi189 podman[79700]: 2024-03-21 12:52:30.380262534 +0000 UTC m=+0.644153353 container remove d2c15ba54b4f9587306441a10cb65af7d311a50b1337c97c68b9f8e5b823b288 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-5, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240319, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True) 2024-03-21T12:52:30.709 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:52:30 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6[62661]: 2024-03-21T12:52:30.480+0000 7fbdb0c72700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:46.295470+0000 front 2024-03-21T12:51:46.295565+0000 (oldest deadline 2024-03-21T12:52:14.496180+0000) 2024-03-21T12:52:30.709 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:52:30 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6[62661]: 2024-03-21T12:52:30.480+0000 7fbdb0c72700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-03-21T12:51:55.597270+0000 front 2024-03-21T12:51:55.597221+0000 (oldest deadline 2024-03-21T12:52:20.297338+0000) 2024-03-21T12:52:30.709 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:52:30 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6[62661]: 2024-03-21T12:52:30.480+0000 7fbdb0c72700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-03-21T12:52:04.398328+0000 front 2024-03-21T12:52:04.398220+0000 (oldest deadline 2024-03-21T12:52:28.498321+0000) 2024-03-21T12:52:30.975 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:52:30 smithi189 podman[79795]: 2024-03-21 12:52:30.611953059 +0000 UTC m=+0.018326523 image pull 19e12293512ff7978c8adb5f72a829060dd47f07bfd6903b6b3178d2ebc29e4e quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93 2024-03-21T12:52:30.976 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:52:30 smithi189 podman[79795]: 2024-03-21 12:52:30.747856865 +0000 UTC m=+0.154230318 container create ab36702b2e660d986a416caed43058191a5fa78c7d550b2bec16145818ee59c3 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-5-deactivate, org.label-schema.build-date=20240319, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-03-21T12:52:30.976 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:52:30 smithi189 podman[79795]: 2024-03-21 12:52:30.975999281 +0000 UTC m=+0.382372744 container init ab36702b2e660d986a416caed43058191a5fa78c7d550b2bec16145818ee59c3 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-5-deactivate, io.buildah.version=1.33.5, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240319, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-21T12:52:30.976 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:30 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:30.883+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:43.900855+0000 front 2024-03-21T12:51:43.900987+0000 (oldest deadline 2024-03-21T12:52:11.501313+0000) 2024-03-21T12:52:30.976 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:30 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:30.883+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-03-21T12:51:51.501922+0000 front 2024-03-21T12:51:51.501788+0000 (oldest deadline 2024-03-21T12:52:17.401661+0000) 2024-03-21T12:52:30.976 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:30 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:30.883+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-03-21T12:52:03.103687+0000 front 2024-03-21T12:52:03.103788+0000 (oldest deadline 2024-03-21T12:52:26.603833+0000) 2024-03-21T12:52:31.271 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:52:30 smithi189 podman[79795]: 2024-03-21 12:52:30.979288646 +0000 UTC m=+0.385662104 container start ab36702b2e660d986a416caed43058191a5fa78c7d550b2bec16145818ee59c3 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-5-deactivate, maintainer=Guillaume Abrioux , RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240319, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30) 2024-03-21T12:52:31.271 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:52:31 smithi189 podman[79795]: 2024-03-21 12:52:31.059155985 +0000 UTC m=+0.465529458 container attach ab36702b2e660d986a416caed43058191a5fa78c7d550b2bec16145818ee59c3 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-5-deactivate, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240319, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, io.buildah.version=1.33.5) 2024-03-21T12:52:31.535 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 21 12:52:31 smithi189 podman[79795]: 2024-03-21 12:52:31.272077541 +0000 UTC m=+0.678450997 container died ab36702b2e660d986a416caed43058191a5fa78c7d550b2bec16145818ee59c3 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-5-deactivate, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240319, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, io.buildah.version=1.33.5, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-03-21T12:52:31.536 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:52:31 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6[62661]: 2024-03-21T12:52:31.446+0000 7fbdb0c72700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:46.295470+0000 front 2024-03-21T12:51:46.295565+0000 (oldest deadline 2024-03-21T12:52:14.496180+0000) 2024-03-21T12:52:31.536 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:52:31 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6[62661]: 2024-03-21T12:52:31.446+0000 7fbdb0c72700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-03-21T12:51:55.597270+0000 front 2024-03-21T12:51:55.597221+0000 (oldest deadline 2024-03-21T12:52:20.297338+0000) 2024-03-21T12:52:31.536 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:52:31 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6[62661]: 2024-03-21T12:52:31.446+0000 7fbdb0c72700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-03-21T12:52:04.398328+0000 front 2024-03-21T12:52:04.398220+0000 (oldest deadline 2024-03-21T12:52:28.498321+0000) 2024-03-21T12:52:31.912 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:31 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:31.880+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:43.900855+0000 front 2024-03-21T12:51:43.900987+0000 (oldest deadline 2024-03-21T12:52:11.501313+0000) 2024-03-21T12:52:31.913 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:31 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:31.880+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-03-21T12:51:51.501922+0000 front 2024-03-21T12:51:51.501788+0000 (oldest deadline 2024-03-21T12:52:17.401661+0000) 2024-03-21T12:52:31.913 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:31 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:31.880+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-03-21T12:52:03.103687+0000 front 2024-03-21T12:52:03.103788+0000 (oldest deadline 2024-03-21T12:52:26.603833+0000) 2024-03-21T12:52:31.937 DEBUG:teuthology.orchestra.run.smithi189:> sudo pkill -f 'journalctl -f -n 0 -u ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715@osd.5.service' 2024-03-21T12:52:31.984 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-21T12:52:31.984 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-03-21T12:52:31.984 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-03-21T12:52:31.984 DEBUG:teuthology.orchestra.run.smithi189:> sudo systemctl stop ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715@osd.6 2024-03-21T12:52:32.282 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:52:32 smithi189 systemd[1]: Stopping Ceph osd.6 for 3fc7f0f2-e77e-11ee-95cd-87774f69a715... 2024-03-21T12:52:32.536 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:52:32 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6[62661]: 2024-03-21T12:52:32.281+0000 7fbdb467e700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-21T12:52:32.536 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:52:32 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6[62661]: 2024-03-21T12:52:32.281+0000 7fbdb467e700 -1 osd.6 164 *** Got signal Terminated *** 2024-03-21T12:52:32.536 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:52:32 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6[62661]: 2024-03-21T12:52:32.281+0000 7fbdb467e700 -1 osd.6 164 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-21T12:52:32.536 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:52:32 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6[62661]: 2024-03-21T12:52:32.398+0000 7fbdb0c72700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:46.295470+0000 front 2024-03-21T12:51:46.295565+0000 (oldest deadline 2024-03-21T12:52:14.496180+0000) 2024-03-21T12:52:32.536 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:52:32 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6[62661]: 2024-03-21T12:52:32.398+0000 7fbdb0c72700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-03-21T12:51:55.597270+0000 front 2024-03-21T12:51:55.597221+0000 (oldest deadline 2024-03-21T12:52:20.297338+0000) 2024-03-21T12:52:32.536 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:52:32 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6[62661]: 2024-03-21T12:52:32.398+0000 7fbdb0c72700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-03-21T12:52:04.398328+0000 front 2024-03-21T12:52:04.398220+0000 (oldest deadline 2024-03-21T12:52:28.498321+0000) 2024-03-21T12:52:33.286 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:32 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:32.875+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:43.900855+0000 front 2024-03-21T12:51:43.900987+0000 (oldest deadline 2024-03-21T12:52:11.501313+0000) 2024-03-21T12:52:33.286 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:32 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:32.875+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-03-21T12:51:51.501922+0000 front 2024-03-21T12:51:51.501788+0000 (oldest deadline 2024-03-21T12:52:17.401661+0000) 2024-03-21T12:52:33.286 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:32 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:32.875+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-03-21T12:52:03.103687+0000 front 2024-03-21T12:52:03.103788+0000 (oldest deadline 2024-03-21T12:52:26.603833+0000) 2024-03-21T12:52:33.698 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:52:33 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6[62661]: 2024-03-21T12:52:33.403+0000 7fbdb0c72700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:46.295470+0000 front 2024-03-21T12:51:46.295565+0000 (oldest deadline 2024-03-21T12:52:14.496180+0000) 2024-03-21T12:52:33.698 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:52:33 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6[62661]: 2024-03-21T12:52:33.403+0000 7fbdb0c72700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-03-21T12:51:55.597270+0000 front 2024-03-21T12:51:55.597221+0000 (oldest deadline 2024-03-21T12:52:20.297338+0000) 2024-03-21T12:52:33.698 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:52:33 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6[62661]: 2024-03-21T12:52:33.403+0000 7fbdb0c72700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-03-21T12:52:04.398328+0000 front 2024-03-21T12:52:04.398220+0000 (oldest deadline 2024-03-21T12:52:28.498321+0000) 2024-03-21T12:52:34.035 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:33 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:33.913+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:43.900855+0000 front 2024-03-21T12:51:43.900987+0000 (oldest deadline 2024-03-21T12:52:11.501313+0000) 2024-03-21T12:52:34.036 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:33 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:33.913+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-03-21T12:51:51.501922+0000 front 2024-03-21T12:51:51.501788+0000 (oldest deadline 2024-03-21T12:52:17.401661+0000) 2024-03-21T12:52:34.036 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:33 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:33.913+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-03-21T12:52:03.103687+0000 front 2024-03-21T12:52:03.103788+0000 (oldest deadline 2024-03-21T12:52:26.603833+0000) 2024-03-21T12:52:34.786 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:52:34 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6[62661]: 2024-03-21T12:52:34.378+0000 7fbdb0c72700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:46.295470+0000 front 2024-03-21T12:51:46.295565+0000 (oldest deadline 2024-03-21T12:52:14.496180+0000) 2024-03-21T12:52:34.786 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:52:34 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6[62661]: 2024-03-21T12:52:34.378+0000 7fbdb0c72700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-03-21T12:51:55.597270+0000 front 2024-03-21T12:51:55.597221+0000 (oldest deadline 2024-03-21T12:52:20.297338+0000) 2024-03-21T12:52:34.786 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:52:34 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6[62661]: 2024-03-21T12:52:34.378+0000 7fbdb0c72700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-03-21T12:52:04.398328+0000 front 2024-03-21T12:52:04.398220+0000 (oldest deadline 2024-03-21T12:52:28.498321+0000) 2024-03-21T12:52:35.285 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:34 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:34.903+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:43.900855+0000 front 2024-03-21T12:51:43.900987+0000 (oldest deadline 2024-03-21T12:52:11.501313+0000) 2024-03-21T12:52:35.286 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:34 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:34.903+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-03-21T12:51:51.501922+0000 front 2024-03-21T12:51:51.501788+0000 (oldest deadline 2024-03-21T12:52:17.401661+0000) 2024-03-21T12:52:35.286 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:34 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:34.903+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-03-21T12:52:03.103687+0000 front 2024-03-21T12:52:03.103788+0000 (oldest deadline 2024-03-21T12:52:26.603833+0000) 2024-03-21T12:52:35.786 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:52:35 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6[62661]: 2024-03-21T12:52:35.382+0000 7fbdb0c72700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:46.295470+0000 front 2024-03-21T12:51:46.295565+0000 (oldest deadline 2024-03-21T12:52:14.496180+0000) 2024-03-21T12:52:35.786 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:52:35 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6[62661]: 2024-03-21T12:52:35.382+0000 7fbdb0c72700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-03-21T12:51:55.597270+0000 front 2024-03-21T12:51:55.597221+0000 (oldest deadline 2024-03-21T12:52:20.297338+0000) 2024-03-21T12:52:35.786 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:52:35 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6[62661]: 2024-03-21T12:52:35.382+0000 7fbdb0c72700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-03-21T12:52:04.398328+0000 front 2024-03-21T12:52:04.398220+0000 (oldest deadline 2024-03-21T12:52:28.498321+0000) 2024-03-21T12:52:35.786 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:52:35 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6[62661]: 2024-03-21T12:52:35.382+0000 7fbdb0c72700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.22:6830 osd.3 since back 2024-03-21T12:52:11.399151+0000 front 2024-03-21T12:52:11.399126+0000 (oldest deadline 2024-03-21T12:52:34.899113+0000) 2024-03-21T12:52:36.286 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:35 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:35.862+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:43.900855+0000 front 2024-03-21T12:51:43.900987+0000 (oldest deadline 2024-03-21T12:52:11.501313+0000) 2024-03-21T12:52:36.286 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:35 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:35.862+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-03-21T12:51:51.501922+0000 front 2024-03-21T12:51:51.501788+0000 (oldest deadline 2024-03-21T12:52:17.401661+0000) 2024-03-21T12:52:36.286 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:35 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:35.862+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-03-21T12:52:03.103687+0000 front 2024-03-21T12:52:03.103788+0000 (oldest deadline 2024-03-21T12:52:26.603833+0000) 2024-03-21T12:52:36.786 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:52:36 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6[62661]: 2024-03-21T12:52:36.403+0000 7fbdb0c72700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:46.295470+0000 front 2024-03-21T12:51:46.295565+0000 (oldest deadline 2024-03-21T12:52:14.496180+0000) 2024-03-21T12:52:36.786 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:52:36 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6[62661]: 2024-03-21T12:52:36.403+0000 7fbdb0c72700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-03-21T12:51:55.597270+0000 front 2024-03-21T12:51:55.597221+0000 (oldest deadline 2024-03-21T12:52:20.297338+0000) 2024-03-21T12:52:36.786 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:52:36 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6[62661]: 2024-03-21T12:52:36.403+0000 7fbdb0c72700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-03-21T12:52:04.398328+0000 front 2024-03-21T12:52:04.398220+0000 (oldest deadline 2024-03-21T12:52:28.498321+0000) 2024-03-21T12:52:36.786 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:52:36 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6[62661]: 2024-03-21T12:52:36.403+0000 7fbdb0c72700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.22:6830 osd.3 since back 2024-03-21T12:52:11.399151+0000 front 2024-03-21T12:52:11.399126+0000 (oldest deadline 2024-03-21T12:52:34.899113+0000) 2024-03-21T12:52:37.285 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:36 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:36.904+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:43.900855+0000 front 2024-03-21T12:51:43.900987+0000 (oldest deadline 2024-03-21T12:52:11.501313+0000) 2024-03-21T12:52:37.286 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:36 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:36.904+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-03-21T12:51:51.501922+0000 front 2024-03-21T12:51:51.501788+0000 (oldest deadline 2024-03-21T12:52:17.401661+0000) 2024-03-21T12:52:37.286 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:36 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:36.904+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-03-21T12:52:03.103687+0000 front 2024-03-21T12:52:03.103788+0000 (oldest deadline 2024-03-21T12:52:26.603833+0000) 2024-03-21T12:52:38.016 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:52:37 smithi189 podman[79901]: 2024-03-21 12:52:37.69837326 +0000 UTC m=+5.524299028 container died 291089fb3596c5b9ee54f27a4b98f147108fa4aebca287ba916c16e5b718a9c8 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6, org.label-schema.license=GPLv2, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.build-date=20240319, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe) 2024-03-21T12:52:38.017 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:37 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:37.942+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:43.900855+0000 front 2024-03-21T12:51:43.900987+0000 (oldest deadline 2024-03-21T12:52:11.501313+0000) 2024-03-21T12:52:38.017 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:37 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:37.942+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-03-21T12:51:51.501922+0000 front 2024-03-21T12:51:51.501788+0000 (oldest deadline 2024-03-21T12:52:17.401661+0000) 2024-03-21T12:52:38.017 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:37 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:37.942+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-03-21T12:52:03.103687+0000 front 2024-03-21T12:52:03.103788+0000 (oldest deadline 2024-03-21T12:52:26.603833+0000) 2024-03-21T12:52:38.017 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:37 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:37.942+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6830 osd.3 since back 2024-03-21T12:52:12.504700+0000 front 2024-03-21T12:52:12.504643+0000 (oldest deadline 2024-03-21T12:52:37.804690+0000) 2024-03-21T12:52:38.285 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:52:38 smithi189 podman[79901]: 2024-03-21 12:52:38.199185585 +0000 UTC m=+6.025111354 container cleanup 291089fb3596c5b9ee54f27a4b98f147108fa4aebca287ba916c16e5b718a9c8 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.build-date=20240319, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-03-21T12:52:38.286 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:52:38 smithi189 bash[79901]: ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6 2024-03-21T12:52:38.701 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:52:38 smithi189 podman[79919]: 2024-03-21 12:52:38.409036405 +0000 UTC m=+0.708736379 container remove 291089fb3596c5b9ee54f27a4b98f147108fa4aebca287ba916c16e5b718a9c8 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.build-date=20240319, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux ) 2024-03-21T12:52:38.995 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:38 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:38.946+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:43.900855+0000 front 2024-03-21T12:51:43.900987+0000 (oldest deadline 2024-03-21T12:52:11.501313+0000) 2024-03-21T12:52:38.995 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:38 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:38.946+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-03-21T12:51:51.501922+0000 front 2024-03-21T12:51:51.501788+0000 (oldest deadline 2024-03-21T12:52:17.401661+0000) 2024-03-21T12:52:38.995 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:38 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:38.946+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-03-21T12:52:03.103687+0000 front 2024-03-21T12:52:03.103788+0000 (oldest deadline 2024-03-21T12:52:26.603833+0000) 2024-03-21T12:52:38.995 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:38 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:38.946+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6830 osd.3 since back 2024-03-21T12:52:12.504700+0000 front 2024-03-21T12:52:12.504643+0000 (oldest deadline 2024-03-21T12:52:37.804690+0000) 2024-03-21T12:52:38.995 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:52:38 smithi189 podman[80015]: 2024-03-21 12:52:38.701445158 +0000 UTC m=+0.114956762 container create 2b5d441f6d629545e777a5e98ac845e4cb75006b5cf29d35d8889ffa28a6499c (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6-deactivate, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240319, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-03-21T12:52:38.995 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:52:38 smithi189 podman[80015]: 2024-03-21 12:52:38.604406161 +0000 UTC m=+0.017917728 image pull 19e12293512ff7978c8adb5f72a829060dd47f07bfd6903b6b3178d2ebc29e4e quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93 2024-03-21T12:52:38.995 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:52:38 smithi189 podman[80015]: 2024-03-21 12:52:38.912488939 +0000 UTC m=+0.326000510 container init 2b5d441f6d629545e777a5e98ac845e4cb75006b5cf29d35d8889ffa28a6499c (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6-deactivate, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.build-date=20240319, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS) 2024-03-21T12:52:38.995 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:52:38 smithi189 podman[80015]: 2024-03-21 12:52:38.915817209 +0000 UTC m=+0.329328780 container start 2b5d441f6d629545e777a5e98ac845e4cb75006b5cf29d35d8889ffa28a6499c (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6-deactivate, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240319) 2024-03-21T12:52:38.995 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:52:38 smithi189 podman[80015]: 2024-03-21 12:52:38.995417843 +0000 UTC m=+0.408929415 container attach 2b5d441f6d629545e777a5e98ac845e4cb75006b5cf29d35d8889ffa28a6499c (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6-deactivate, org.label-schema.build-date=20240319, ceph=True, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe) 2024-03-21T12:52:39.286 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 21 12:52:39 smithi189 podman[80015]: 2024-03-21 12:52:39.201935677 +0000 UTC m=+0.615447244 container died 2b5d441f6d629545e777a5e98ac845e4cb75006b5cf29d35d8889ffa28a6499c (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-6-deactivate, org.label-schema.build-date=20240319, ceph=True, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.schema-version=1.0, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5) 2024-03-21T12:52:39.844 DEBUG:teuthology.orchestra.run.smithi189:> sudo pkill -f 'journalctl -f -n 0 -u ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715@osd.6.service' 2024-03-21T12:52:39.884 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-21T12:52:39.884 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-03-21T12:52:39.884 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-03-21T12:52:39.884 DEBUG:teuthology.orchestra.run.smithi189:> sudo systemctl stop ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715@osd.7 2024-03-21T12:52:40.219 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:39 smithi189 systemd[1]: Stopping Ceph osd.7 for 3fc7f0f2-e77e-11ee-95cd-87774f69a715... 2024-03-21T12:52:40.219 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:39 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:39.985+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:43.900855+0000 front 2024-03-21T12:51:43.900987+0000 (oldest deadline 2024-03-21T12:52:11.501313+0000) 2024-03-21T12:52:40.219 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:39 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:39.985+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-03-21T12:51:51.501922+0000 front 2024-03-21T12:51:51.501788+0000 (oldest deadline 2024-03-21T12:52:17.401661+0000) 2024-03-21T12:52:40.219 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:39 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:39.985+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-03-21T12:52:03.103687+0000 front 2024-03-21T12:52:03.103788+0000 (oldest deadline 2024-03-21T12:52:26.603833+0000) 2024-03-21T12:52:40.219 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:39 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:39.985+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6830 osd.3 since back 2024-03-21T12:52:12.504700+0000 front 2024-03-21T12:52:12.504643+0000 (oldest deadline 2024-03-21T12:52:37.804690+0000) 2024-03-21T12:52:40.219 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:40 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:40.218+0000 7fe278e33700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-21T12:52:40.219 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:40 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:40.218+0000 7fe278e33700 -1 osd.7 164 *** Got signal Terminated *** 2024-03-21T12:52:40.219 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:40 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:40.218+0000 7fe278e33700 -1 osd.7 164 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-21T12:52:41.286 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:40 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:40.941+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:43.900855+0000 front 2024-03-21T12:51:43.900987+0000 (oldest deadline 2024-03-21T12:52:11.501313+0000) 2024-03-21T12:52:41.286 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:40 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:40.941+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-03-21T12:51:51.501922+0000 front 2024-03-21T12:51:51.501788+0000 (oldest deadline 2024-03-21T12:52:17.401661+0000) 2024-03-21T12:52:41.286 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:40 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:40.941+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-03-21T12:52:03.103687+0000 front 2024-03-21T12:52:03.103788+0000 (oldest deadline 2024-03-21T12:52:26.603833+0000) 2024-03-21T12:52:41.286 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:40 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:40.941+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6830 osd.3 since back 2024-03-21T12:52:12.504700+0000 front 2024-03-21T12:52:12.504643+0000 (oldest deadline 2024-03-21T12:52:37.804690+0000) 2024-03-21T12:52:42.286 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:41 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:41.988+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:43.900855+0000 front 2024-03-21T12:51:43.900987+0000 (oldest deadline 2024-03-21T12:52:11.501313+0000) 2024-03-21T12:52:42.286 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:41 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:41.988+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-03-21T12:51:51.501922+0000 front 2024-03-21T12:51:51.501788+0000 (oldest deadline 2024-03-21T12:52:17.401661+0000) 2024-03-21T12:52:42.286 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:41 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:41.988+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-03-21T12:52:03.103687+0000 front 2024-03-21T12:52:03.103788+0000 (oldest deadline 2024-03-21T12:52:26.603833+0000) 2024-03-21T12:52:42.286 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:41 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:41.988+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6830 osd.3 since back 2024-03-21T12:52:12.504700+0000 front 2024-03-21T12:52:12.504643+0000 (oldest deadline 2024-03-21T12:52:37.804690+0000) 2024-03-21T12:52:42.286 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:41 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:41.988+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.189:6804 osd.4 since back 2024-03-21T12:52:17.805042+0000 front 2024-03-21T12:52:17.805198+0000 (oldest deadline 2024-03-21T12:52:41.905010+0000) 2024-03-21T12:52:43.286 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:43 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:43.011+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:43.900855+0000 front 2024-03-21T12:51:43.900987+0000 (oldest deadline 2024-03-21T12:52:11.501313+0000) 2024-03-21T12:52:43.286 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:43 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:43.011+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-03-21T12:51:51.501922+0000 front 2024-03-21T12:51:51.501788+0000 (oldest deadline 2024-03-21T12:52:17.401661+0000) 2024-03-21T12:52:43.286 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:43 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:43.011+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-03-21T12:52:03.103687+0000 front 2024-03-21T12:52:03.103788+0000 (oldest deadline 2024-03-21T12:52:26.603833+0000) 2024-03-21T12:52:43.286 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:43 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:43.011+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6830 osd.3 since back 2024-03-21T12:52:12.504700+0000 front 2024-03-21T12:52:12.504643+0000 (oldest deadline 2024-03-21T12:52:37.804690+0000) 2024-03-21T12:52:43.286 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:43 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:43.011+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.189:6804 osd.4 since back 2024-03-21T12:52:17.805042+0000 front 2024-03-21T12:52:17.805198+0000 (oldest deadline 2024-03-21T12:52:41.905010+0000) 2024-03-21T12:52:44.286 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:43 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:43.984+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:43.900855+0000 front 2024-03-21T12:51:43.900987+0000 (oldest deadline 2024-03-21T12:52:11.501313+0000) 2024-03-21T12:52:44.286 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:43 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:43.984+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-03-21T12:51:51.501922+0000 front 2024-03-21T12:51:51.501788+0000 (oldest deadline 2024-03-21T12:52:17.401661+0000) 2024-03-21T12:52:44.286 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:43 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:43.984+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-03-21T12:52:03.103687+0000 front 2024-03-21T12:52:03.103788+0000 (oldest deadline 2024-03-21T12:52:26.603833+0000) 2024-03-21T12:52:44.286 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:43 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:43.984+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6830 osd.3 since back 2024-03-21T12:52:12.504700+0000 front 2024-03-21T12:52:12.504643+0000 (oldest deadline 2024-03-21T12:52:37.804690+0000) 2024-03-21T12:52:44.286 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:43 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:43.984+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.189:6804 osd.4 since back 2024-03-21T12:52:17.805042+0000 front 2024-03-21T12:52:17.805198+0000 (oldest deadline 2024-03-21T12:52:41.905010+0000) 2024-03-21T12:52:45.286 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:44 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:44.941+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-03-21T12:51:43.900855+0000 front 2024-03-21T12:51:43.900987+0000 (oldest deadline 2024-03-21T12:52:11.501313+0000) 2024-03-21T12:52:45.286 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:44 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:44.941+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-03-21T12:51:51.501922+0000 front 2024-03-21T12:51:51.501788+0000 (oldest deadline 2024-03-21T12:52:17.401661+0000) 2024-03-21T12:52:45.286 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:44 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:44.941+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-03-21T12:52:03.103687+0000 front 2024-03-21T12:52:03.103788+0000 (oldest deadline 2024-03-21T12:52:26.603833+0000) 2024-03-21T12:52:45.286 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:44 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:44.941+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.22:6830 osd.3 since back 2024-03-21T12:52:12.504700+0000 front 2024-03-21T12:52:12.504643+0000 (oldest deadline 2024-03-21T12:52:37.804690+0000) 2024-03-21T12:52:45.286 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:44 smithi189 ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7[64896]: 2024-03-21T12:52:44.941+0000 7fe27141f700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.189:6804 osd.4 since back 2024-03-21T12:52:17.805042+0000 front 2024-03-21T12:52:17.805198+0000 (oldest deadline 2024-03-21T12:52:41.905010+0000) 2024-03-21T12:52:46.035 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:45 smithi189 podman[80119]: 2024-03-21 12:52:45.755747996 +0000 UTC m=+5.649743399 container died 247b82310a8a7c728b422e3d15a1ce8b335d605536663f694ed953d0d54bd2e3 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240319, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, ceph=True) 2024-03-21T12:52:46.386 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:46 smithi189 podman[80119]: 2024-03-21 12:52:46.118529338 +0000 UTC m=+6.012524743 container cleanup 247b82310a8a7c728b422e3d15a1ce8b335d605536663f694ed953d0d54bd2e3 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.build-date=20240319, CEPH_POINT_RELEASE=, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-21T12:52:46.386 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:46 smithi189 bash[80119]: ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7 2024-03-21T12:52:46.738 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:46 smithi189 podman[80180]: 2024-03-21 12:52:46.386269283 +0000 UTC m=+0.623157448 container remove 247b82310a8a7c728b422e3d15a1ce8b335d605536663f694ed953d0d54bd2e3 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7, org.label-schema.build-date=20240319, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_CLEAN=True, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_BRANCH=HEAD) 2024-03-21T12:52:47.022 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:46 smithi189 podman[80274]: 2024-03-21 12:52:46.641464976 +0000 UTC m=+0.017718034 image pull 19e12293512ff7978c8adb5f72a829060dd47f07bfd6903b6b3178d2ebc29e4e quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93 2024-03-21T12:52:47.023 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:46 smithi189 podman[80274]: 2024-03-21 12:52:46.895479059 +0000 UTC m=+0.271732094 container create 7848e50dc6e4f6188644c05e7c26ef4e5dda0c5d7f532a9870b555737ad249e4 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7-deactivate, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240319, ceph=True, maintainer=Guillaume Abrioux , RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-03-21T12:52:47.286 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:47 smithi189 podman[80274]: 2024-03-21 12:52:47.123150591 +0000 UTC m=+0.499403649 container init 7848e50dc6e4f6188644c05e7c26ef4e5dda0c5d7f532a9870b555737ad249e4 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240319, io.buildah.version=1.33.5, ceph=True, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30) 2024-03-21T12:52:47.286 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:47 smithi189 podman[80274]: 2024-03-21 12:52:47.126219496 +0000 UTC m=+0.502472533 container start 7848e50dc6e4f6188644c05e7c26ef4e5dda0c5d7f532a9870b555737ad249e4 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7-deactivate, org.label-schema.build-date=20240319, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5) 2024-03-21T12:52:47.286 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:47 smithi189 podman[80274]: 2024-03-21 12:52:47.206484237 +0000 UTC m=+0.582737286 container attach 7848e50dc6e4f6188644c05e7c26ef4e5dda0c5d7f532a9870b555737ad249e4 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7-deactivate, GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, org.label-schema.build-date=20240319, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.33.5) 2024-03-21T12:52:47.721 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 21 12:52:47 smithi189 podman[80274]: 2024-03-21 12:52:47.416610249 +0000 UTC m=+0.792863286 container died 7848e50dc6e4f6188644c05e7c26ef4e5dda0c5d7f532a9870b555737ad249e4 (image=quay.ceph.io/ceph-ci/ceph@sha256:35f4b82cbae750007a45d4eba3ff9f653c457c283782141ef353684823d81e93, name=ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715-osd-7-deactivate, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240319, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, RELEASE=wip-guits-testing-2024-03-21-0818-960fbbe, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-03-21T12:52:48.011 DEBUG:teuthology.orchestra.run.smithi189:> sudo pkill -f 'journalctl -f -n 0 -u ceph-3fc7f0f2-e77e-11ee-95cd-87774f69a715@osd.7.service' 2024-03-21T12:52:48.054 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-21T12:52:48.054 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-03-21T12:52:48.054 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 --force --keep-logs 2024-03-21T12:55:46.632 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 --force --keep-logs 2024-03-21T12:56:06.334 DEBUG:teuthology.orchestra.run.smithi022:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-03-21T12:56:06.371 DEBUG:teuthology.orchestra.run.smithi189:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-03-21T12:56:06.400 INFO:tasks.cephadm:Archiving crash dumps... 2024-03-21T12:56:06.403 DEBUG:teuthology.misc:Transferring archived files from smithi022:/var/lib/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/crash to /home/teuthworker/archive/gabrioux-2024-03-21_12:04:17-orch:cephadm-wip-guits-testing-2024-03-21-0818-distro-default-smithi/7614906/remote/smithi022/crash 2024-03-21T12:56:06.404 DEBUG:teuthology.orchestra.run.smithi022:> sudo tar c -f - -C /var/lib/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/crash -- . 2024-03-21T12:56:06.481 INFO:teuthology.orchestra.run.smithi022.stderr:tar: /var/lib/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/crash: Cannot open: No such file or directory 2024-03-21T12:56:06.482 INFO:teuthology.orchestra.run.smithi022.stderr:tar: Error is not recoverable: exiting now 2024-03-21T12:56:06.484 DEBUG:teuthology.misc:Transferring archived files from smithi189:/var/lib/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/crash to /home/teuthworker/archive/gabrioux-2024-03-21_12:04:17-orch:cephadm-wip-guits-testing-2024-03-21-0818-distro-default-smithi/7614906/remote/smithi189/crash 2024-03-21T12:56:06.485 DEBUG:teuthology.orchestra.run.smithi189:> sudo tar c -f - -C /var/lib/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/crash -- . 2024-03-21T12:56:06.524 INFO:teuthology.orchestra.run.smithi189.stderr:tar: /var/lib/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/crash: Cannot open: No such file or directory 2024-03-21T12:56:06.524 INFO:teuthology.orchestra.run.smithi189.stderr:tar: Error is not recoverable: exiting now 2024-03-21T12:56:06.525 INFO:tasks.cephadm:Checking cluster log for badness... 2024-03-21T12:56:06.525 DEBUG:teuthology.orchestra.run.smithi022:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-03-21T12:56:06.556 INFO:teuthology.orchestra.run.smithi022.stdout:2024-03-21T12:32:21.542288+0000 mon.smithi022 (mon.0) 495 : cluster [WRN] Health check failed: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded (PG_DEGRADED) 2024-03-21T12:56:06.558 WARNING:tasks.cephadm:Found errors (ERR|WRN|SEC) in cluster log 2024-03-21T12:56:06.558 DEBUG:teuthology.orchestra.run.smithi022:> sudo egrep '\[SEC\]' /var/log/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-03-21T12:56:06.631 DEBUG:teuthology.orchestra.run.smithi022:> sudo egrep '\[ERR\]' /var/log/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-03-21T12:56:06.701 DEBUG:teuthology.orchestra.run.smithi022:> sudo egrep '\[WRN\]' /var/log/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-03-21T12:56:06.771 INFO:teuthology.orchestra.run.smithi022.stdout:2024-03-21T12:32:21.542288+0000 mon.smithi022 (mon.0) 495 : cluster [WRN] Health check failed: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded (PG_DEGRADED) 2024-03-21T12:56:06.773 INFO:tasks.cephadm:Compressing logs... 2024-03-21T12:56:06.773 DEBUG:teuthology.orchestra.run.smithi022:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-03-21T12:56:06.816 DEBUG:teuthology.orchestra.run.smithi189:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-03-21T12:56:06.841 INFO:teuthology.orchestra.run.smithi022.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-03-21T12:56:06.841 INFO:teuthology.orchestra.run.smithi022.stderr:: No such file or directory 2024-03-21T12:56:06.842 INFO:teuthology.orchestra.run.smithi022.stderr:gzip -5 --verbose -- /var/log/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/ceph.audit.log 2024-03-21T12:56:06.842 INFO:teuthology.orchestra.run.smithi022.stderr:gzip -5 --verbose -- /var/log/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/ceph-osd.1.log 2024-03-21T12:56:06.842 INFO:teuthology.orchestra.run.smithi022.stderr:/var/log/ceph/cephadm.log: gzip -5 --verbose -- /var/log/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/ceph-mon.smithi022.log 2024-03-21T12:56:06.842 INFO:teuthology.orchestra.run.smithi022.stderr:/var/log/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/ceph.audit.log: /var/log/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/ceph-mds.foofs.smithi022.bcnxho.log 2024-03-21T12:56:06.845 INFO:teuthology.orchestra.run.smithi022.stderr:gzip -5 --verbose -- /var/log/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/ceph-osd.3.log 2024-03-21T12:56:06.845 INFO:teuthology.orchestra.run.smithi022.stderr:/var/log/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/ceph-mon.smithi022.log: /var/log/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/ceph-mds.foofs.smithi022.bcnxho.log: gzip -5 --verbose -- /var/log/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/ceph-mgr.smithi022.fektzs.log 2024-03-21T12:56:06.845 INFO:teuthology.orchestra.run.smithi022.stderr:/var/log/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/ceph-osd.3.log: gzip 89.1% -- replaced with /var/log/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/ceph-mds.foofs.smithi022.bcnxho.log.gz 2024-03-21T12:56:06.845 INFO:teuthology.orchestra.run.smithi022.stderr: -5 --verbose -- /var/log/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/ceph-osd.0.log 2024-03-21T12:56:06.845 INFO:teuthology.orchestra.run.smithi022.stderr:/var/log/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/ceph-mgr.smithi022.fektzs.log: gzip -5 --verbose -- /var/log/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/ceph.cephadm.log 2024-03-21T12:56:06.845 INFO:teuthology.orchestra.run.smithi022.stderr:/var/log/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/ceph-osd.0.log: gzip -5 --verbose -- /var/log/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/ceph.log 2024-03-21T12:56:06.845 INFO:teuthology.orchestra.run.smithi022.stderr:/var/log/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/ceph.cephadm.log: 85.9% -- replaced with /var/log/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/ceph.cephadm.log.gz 2024-03-21T12:56:06.845 INFO:teuthology.orchestra.run.smithi022.stderr:gzip -5 --verbose -- /var/log/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/ceph-osd.2.log 2024-03-21T12:56:06.847 INFO:teuthology.orchestra.run.smithi189.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-03-21T12:56:06.847 INFO:teuthology.orchestra.run.smithi189.stderr:: No such file or directory 2024-03-21T12:56:06.847 INFO:teuthology.orchestra.run.smithi189.stderr:gzip -5 --verbose -- /var/log/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/ceph-osd.6.log 2024-03-21T12:56:06.847 INFO:teuthology.orchestra.run.smithi189.stderr:/var/log/ceph/cephadm.log: gzip -5 --verbose -- /var/log/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/ceph.audit.log 2024-03-21T12:56:06.847 INFO:teuthology.orchestra.run.smithi189.stderr:/var/log/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/ceph-osd.6.log: gzip -5 --verbose -- /var/log/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/ceph-osd.4.log 2024-03-21T12:56:06.848 INFO:teuthology.orchestra.run.smithi022.stderr:/var/log/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/ceph.log: 91.6% -- replaced with /var/log/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/ceph.audit.log.gz 2024-03-21T12:56:06.852 INFO:teuthology.orchestra.run.smithi189.stderr:/var/log/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/ceph-mon.smithi189.log 2024-03-21T12:56:06.853 INFO:teuthology.orchestra.run.smithi189.stderr:gzip -5 --verbose -- /var/log/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/ceph-osd.7.log 2024-03-21T12:56:06.853 INFO:teuthology.orchestra.run.smithi189.stderr:/var/log/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/ceph-osd.4.log: /var/log/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/ceph-mon.smithi189.log: gzip -5 --verbose -- /var/log/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/ceph-osd.5.log 2024-03-21T12:56:06.853 INFO:teuthology.orchestra.run.smithi189.stderr:/var/log/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/ceph-osd.7.log: gzip -5 --verbose -- /var/log/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/ceph-mds.foofs.smithi189.lluvko.log 2024-03-21T12:56:06.853 INFO:teuthology.orchestra.run.smithi189.stderr:/var/log/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/ceph-osd.5.log: gzip -5 --verbose -- /var/log/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/ceph.cephadm.log 2024-03-21T12:56:06.853 INFO:teuthology.orchestra.run.smithi189.stderr:/var/log/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/ceph-mds.foofs.smithi189.lluvko.log: 87.8% -- replaced with /var/log/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/ceph-mds.foofs.smithi189.lluvko.log.gz 2024-03-21T12:56:06.853 INFO:teuthology.orchestra.run.smithi189.stderr:gzip -5 --verbose -- /var/log/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/ceph.log 2024-03-21T12:56:06.853 INFO:teuthology.orchestra.run.smithi189.stderr:/var/log/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/ceph.cephadm.log: 85.8% -- replaced with /var/log/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/ceph.cephadm.log.gz 2024-03-21T12:56:06.853 INFO:teuthology.orchestra.run.smithi189.stderr:gzip 91.7% -- replaced with /var/log/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/ceph.audit.log.gz 2024-03-21T12:56:06.853 INFO:teuthology.orchestra.run.smithi189.stderr: -5 --verbose -- /var/log/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/ceph-volume.log 2024-03-21T12:56:06.853 INFO:teuthology.orchestra.run.smithi189.stderr:/var/log/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/ceph.log: gzip -5 --verbose -- /var/log/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/ceph-mgr.smithi189.cnxzvu.log 2024-03-21T12:56:06.853 INFO:teuthology.orchestra.run.smithi189.stderr:/var/log/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/ceph-volume.log: 90.2% -- replaced with /var/log/ceph/cephadm.log.gz 2024-03-21T12:56:06.853 INFO:teuthology.orchestra.run.smithi189.stderr: 87.4% -- replaced with /var/log/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/ceph.log.gz 2024-03-21T12:56:06.854 INFO:teuthology.orchestra.run.smithi022.stderr: 87.4% -- replaced with /var/log/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/ceph.log.gz 2024-03-21T12:56:06.854 INFO:teuthology.orchestra.run.smithi022.stderr:gzip -5 --verbose -- /var/log/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/ceph-volume.log 2024-03-21T12:56:06.864 INFO:teuthology.orchestra.run.smithi189.stderr:/var/log/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/ceph-mgr.smithi189.cnxzvu.log: 91.4% -- replaced with /var/log/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/ceph-volume.log.gz 2024-03-21T12:56:06.869 INFO:teuthology.orchestra.run.smithi022.stderr:/var/log/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/ceph-osd.2.log: /var/log/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/ceph-volume.log: 91.4% -- replaced with /var/log/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/ceph-volume.log.gz 2024-03-21T12:56:06.880 INFO:teuthology.orchestra.run.smithi022.stderr: 91.6% -- replaced with /var/log/ceph/cephadm.log.gz 2024-03-21T12:56:06.905 INFO:teuthology.orchestra.run.smithi189.stderr: 89.7% -- replaced with /var/log/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/ceph-mgr.smithi189.cnxzvu.log.gz 2024-03-21T12:56:07.136 INFO:teuthology.orchestra.run.smithi022.stderr: 88.3% -- replaced with /var/log/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/ceph-mgr.smithi022.fektzs.log.gz 2024-03-21T12:56:07.203 INFO:teuthology.orchestra.run.smithi189.stderr: 92.6% -- replaced with /var/log/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/ceph-mon.smithi189.log.gz 2024-03-21T12:56:08.227 INFO:teuthology.orchestra.run.smithi022.stderr: 91.9% -- replaced with /var/log/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/ceph-mon.smithi022.log.gz 2024-03-21T12:56:20.323 INFO:teuthology.orchestra.run.smithi022.stderr: 94.5% -- replaced with /var/log/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/ceph-osd.2.log.gz 2024-03-21T12:56:21.946 INFO:teuthology.orchestra.run.smithi189.stderr: 93.9% -- replaced with /var/log/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/ceph-osd.4.log.gz 2024-03-21T12:56:24.651 INFO:teuthology.orchestra.run.smithi189.stderr: 94.3% -- replaced with /var/log/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/ceph-osd.6.log.gz 2024-03-21T12:56:25.126 INFO:teuthology.orchestra.run.smithi022.stderr: 94.1% -- replaced with /var/log/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/ceph-osd.0.log.gz 2024-03-21T12:56:25.557 INFO:teuthology.orchestra.run.smithi022.stderr: 94.2% -- replaced with /var/log/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/ceph-osd.1.log.gz 2024-03-21T12:56:27.533 INFO:teuthology.orchestra.run.smithi189.stderr: 94.2% -- replaced with /var/log/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/ceph-osd.5.log.gz 2024-03-21T12:56:27.605 INFO:teuthology.orchestra.run.smithi022.stderr: 94.1% -- replaced with /var/log/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/ceph-osd.3.log.gz 2024-03-21T12:56:27.607 INFO:teuthology.orchestra.run.smithi022.stderr: 2024-03-21T12:56:27.608 INFO:teuthology.orchestra.run.smithi022.stderr:real 0m20.779s 2024-03-21T12:56:27.608 INFO:teuthology.orchestra.run.smithi022.stderr:user 1m10.348s 2024-03-21T12:56:27.608 INFO:teuthology.orchestra.run.smithi022.stderr:sys 0m2.535s 2024-03-21T12:56:32.204 INFO:teuthology.orchestra.run.smithi189.stderr: 94.1% -- replaced with /var/log/ceph/3fc7f0f2-e77e-11ee-95cd-87774f69a715/ceph-osd.7.log.gz 2024-03-21T12:56:32.206 INFO:teuthology.orchestra.run.smithi189.stderr: 2024-03-21T12:56:32.206 INFO:teuthology.orchestra.run.smithi189.stderr:real 0m25.376s 2024-03-21T12:56:32.206 INFO:teuthology.orchestra.run.smithi189.stderr:user 1m16.242s 2024-03-21T12:56:32.206 INFO:teuthology.orchestra.run.smithi189.stderr:sys 0m3.028s 2024-03-21T12:56:32.207 INFO:tasks.cephadm:Archiving logs... 2024-03-21T12:56:32.207 DEBUG:teuthology.misc:Transferring archived files from smithi022:/var/log/ceph to /home/teuthworker/archive/gabrioux-2024-03-21_12:04:17-orch:cephadm-wip-guits-testing-2024-03-21-0818-distro-default-smithi/7614906/remote/smithi022/log 2024-03-21T12:56:32.208 DEBUG:teuthology.orchestra.run.smithi022:> sudo tar c -f - -C /var/log/ceph -- . 2024-03-21T12:56:41.302 DEBUG:teuthology.misc:Transferring archived files from smithi189:/var/log/ceph to /home/teuthworker/archive/gabrioux-2024-03-21_12:04:17-orch:cephadm-wip-guits-testing-2024-03-21-0818-distro-default-smithi/7614906/remote/smithi189/log 2024-03-21T12:56:41.303 DEBUG:teuthology.orchestra.run.smithi189:> sudo tar c -f - -C /var/log/ceph -- . 2024-03-21T12:56:51.629 INFO:tasks.cephadm:Removing cluster... 2024-03-21T12:56:51.629 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 --force 2024-03-21T12:56:52.007 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 3fc7f0f2-e77e-11ee-95cd-87774f69a715 --force 2024-03-21T12:56:52.397 INFO:tasks.cephadm:Removing cephadm ... 2024-03-21T12:56:52.398 DEBUG:teuthology.orchestra.run.smithi022:> rm -rf /home/ubuntu/cephtest/cephadm 2024-03-21T12:56:52.413 DEBUG:teuthology.orchestra.run.smithi189:> rm -rf /home/ubuntu/cephtest/cephadm 2024-03-21T12:56:52.428 INFO:tasks.cephadm:Teardown complete 2024-03-21T12:56:52.428 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-03-21T12:56:52.439 INFO:teuthology.task.clock:Checking final clock skew... 2024-03-21T12:56:52.439 DEBUG:teuthology.orchestra.run.smithi022:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-03-21T12:56:52.456 DEBUG:teuthology.orchestra.run.smithi189:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-03-21T12:56:52.467 INFO:teuthology.orchestra.run.smithi022.stderr:bash: line 1: ntpq: command not found 2024-03-21T12:56:52.475 INFO:teuthology.orchestra.run.smithi022.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-03-21T12:56:52.475 INFO:teuthology.orchestra.run.smithi022.stdout:=============================================================================== 2024-03-21T12:56:52.476 INFO:teuthology.orchestra.run.smithi022.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 5 +1188us[+1179us] +/- 83ms 2024-03-21T12:56:52.476 INFO:teuthology.orchestra.run.smithi022.stdout:^* hv02.front.sepia.ceph.com 3 6 377 2 -1236us[-1246us] +/- 52ms 2024-03-21T12:56:52.476 INFO:teuthology.orchestra.run.smithi022.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 4 +509us[ +500us] +/- 65ms 2024-03-21T12:56:52.476 INFO:teuthology.orchestra.run.smithi022.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-03-21T12:56:52.481 INFO:teuthology.orchestra.run.smithi189.stderr:bash: line 1: ntpq: command not found 2024-03-21T12:56:52.489 INFO:teuthology.orchestra.run.smithi189.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-03-21T12:56:52.489 INFO:teuthology.orchestra.run.smithi189.stdout:=============================================================================== 2024-03-21T12:56:52.489 INFO:teuthology.orchestra.run.smithi189.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 5 +1215us[+1248us] +/- 83ms 2024-03-21T12:56:52.489 INFO:teuthology.orchestra.run.smithi189.stdout:^* hv02.front.sepia.ceph.com 3 6 377 4 -1209us[-1176us] +/- 52ms 2024-03-21T12:56:52.489 INFO:teuthology.orchestra.run.smithi189.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 3 +535us[ +535us] +/- 65ms 2024-03-21T12:56:52.489 INFO:teuthology.orchestra.run.smithi189.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-03-21T12:56:52.490 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-03-21T12:56:52.499 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-03-21T12:56:52.500 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-03-21T12:56:52.509 DEBUG:teuthology.orchestra.run.smithi022:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-03-21T12:56:52.603 DEBUG:teuthology.orchestra.run.smithi189:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-03-21T12:56:52.681 DEBUG:teuthology.orchestra.run.smithi022:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-03-21T12:56:52.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-21T12:56:52.712 DEBUG:teuthology.orchestra.run.smithi189:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-03-21T12:56:52.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-21T12:56:52.743 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-03-21T12:56:52.752 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-03-21T12:56:52.763 INFO:teuthology.task.internal:Duration was 2081.762019 seconds 2024-03-21T12:56:52.764 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-03-21T12:56:52.773 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-03-21T12:56:52.773 DEBUG:teuthology.orchestra.run.smithi022:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-03-21T12:56:52.775 DEBUG:teuthology.orchestra.run.smithi189:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-03-21T12:56:52.816 INFO:teuthology.orchestra.run.smithi022.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-03-21T12:56:52.826 INFO:teuthology.orchestra.run.smithi189.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-03-21T12:56:53.137 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-03-21T12:56:53.137 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi022.front.sepia.ceph.com 2024-03-21T12:56:53.137 DEBUG:teuthology.orchestra.run.smithi022:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-03-21T12:56:53.196 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi189.front.sepia.ceph.com 2024-03-21T12:56:53.196 DEBUG:teuthology.orchestra.run.smithi189:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-03-21T12:56:53.239 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-03-21T12:56:53.239 DEBUG:teuthology.orchestra.run.smithi022:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-03-21T12:56:53.241 DEBUG:teuthology.orchestra.run.smithi189:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-03-21T12:56:53.307 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-03-21T12:56:53.308 DEBUG:teuthology.orchestra.run.smithi022:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-03-21T12:56:53.310 DEBUG:teuthology.orchestra.run.smithi189:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-03-21T12:57:07.079 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-03-21T12:57:07.089 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-03-21T12:57:07.090 DEBUG:teuthology.orchestra.run.smithi022:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-03-21T12:57:07.116 DEBUG:teuthology.orchestra.run.smithi189:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-03-21T12:57:07.149 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-03-21T12:57:07.166 DEBUG:teuthology.orchestra.run.smithi022:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-03-21T12:57:07.172 DEBUG:teuthology.orchestra.run.smithi189:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-03-21T12:57:07.195 INFO:teuthology.orchestra.run.smithi022.stdout:kernel.core_pattern = core 2024-03-21T12:57:07.220 INFO:teuthology.orchestra.run.smithi189.stdout:kernel.core_pattern = core 2024-03-21T12:57:07.251 DEBUG:teuthology.orchestra.run.smithi022:> test -e /home/ubuntu/cephtest/archive/coredump 2024-03-21T12:57:07.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-21T12:57:07.272 DEBUG:teuthology.orchestra.run.smithi189:> test -e /home/ubuntu/cephtest/archive/coredump 2024-03-21T12:57:07.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-21T12:57:07.306 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-03-21T12:57:07.317 INFO:teuthology.task.internal:Transferring archived files... 2024-03-21T12:57:07.318 DEBUG:teuthology.misc:Transferring archived files from smithi022:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/gabrioux-2024-03-21_12:04:17-orch:cephadm-wip-guits-testing-2024-03-21-0818-distro-default-smithi/7614906/remote/smithi022 2024-03-21T12:57:07.319 DEBUG:teuthology.orchestra.run.smithi022:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-03-21T12:57:07.379 DEBUG:teuthology.misc:Transferring archived files from smithi189:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/gabrioux-2024-03-21_12:04:17-orch:cephadm-wip-guits-testing-2024-03-21-0818-distro-default-smithi/7614906/remote/smithi189 2024-03-21T12:57:07.379 DEBUG:teuthology.orchestra.run.smithi189:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-03-21T12:57:07.421 INFO:teuthology.task.internal:Removing archive directory... 2024-03-21T12:57:07.421 DEBUG:teuthology.orchestra.run.smithi022:> rm -rf -- /home/ubuntu/cephtest/archive 2024-03-21T12:57:07.424 DEBUG:teuthology.orchestra.run.smithi189:> rm -rf -- /home/ubuntu/cephtest/archive 2024-03-21T12:57:07.470 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-03-21T12:57:07.481 INFO:teuthology.task.internal:Not uploading archives. 2024-03-21T12:57:07.481 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-03-21T12:57:07.491 INFO:teuthology.task.internal:Tidying up after the test... 2024-03-21T12:57:07.491 DEBUG:teuthology.orchestra.run.smithi022:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-03-21T12:57:07.494 DEBUG:teuthology.orchestra.run.smithi189:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-03-21T12:57:07.505 INFO:teuthology.orchestra.run.smithi022.stdout: 266039 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Mar 21 12:57 /home/ubuntu/cephtest 2024-03-21T12:57:07.525 INFO:teuthology.orchestra.run.smithi189.stdout: 265305 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Mar 21 12:57 /home/ubuntu/cephtest 2024-03-21T12:57:07.526 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-03-21T12:57:07.536 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-03-21T12:57:07.556 INFO:teuthology.nuke:Checking targets against current locks 2024-03-21T12:57:07.584 DEBUG:teuthology.nuke:shortname: smithi022 2024-03-21T12:57:07.584 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-03-21T12:57:07.607 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi022.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/gabrioux-2024-03-21_12:04:17-orch:cephadm-wip-guits-testing-2024-03-21-0818-distro-default-smithi/7614906', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-03-21 12:12:13.938943', 'locked_by': 'scheduled_gabrioux@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNFK97Gjbcjkog67MSuD2palHNl1CaoatkSnBfnz9+hmxYkihXU5tCdYaXONFICPAVaz/2hoRyUkE1ipRratRvs='} 2024-03-21T12:57:07.611 DEBUG:teuthology.nuke:shortname: smithi189 2024-03-21T12:57:07.611 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-03-21T12:57:07.634 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi189.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/gabrioux-2024-03-21_12:04:17-orch:cephadm-wip-guits-testing-2024-03-21-0818-distro-default-smithi/7614906', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-03-21 12:12:13.940154', 'locked_by': 'scheduled_gabrioux@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFBzxp04flUe9qN0scflFfrmoI5QsLDu1LV/+YvcQGrlB/wdkJlgGkGZe2WGMvHTr7uT0lZPSwP1zDMmjuAIkLc='} 2024-03-21T12:57:07.663 INFO:teuthology.orchestra.console.smithi022:Power off 2024-03-21T12:57:07.663 DEBUG:teuthology.orchestra.console.smithi022:pexpect command: ipmitool -H smithi022.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-03-21T12:57:07.687 DEBUG:teuthology.orchestra.console.smithi022:power off output: Chassis Power Control: Down/Off 2024-03-21T12:57:07.688 DEBUG:teuthology.orchestra.console.smithi022:pexpect command: ipmitool -H smithi022.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-03-21T12:57:07.699 INFO:teuthology.orchestra.console.smithi189:Power off 2024-03-21T12:57:07.700 DEBUG:teuthology.orchestra.console.smithi189:pexpect command: ipmitool -H smithi189.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-03-21T12:57:07.711 DEBUG:teuthology.orchestra.console.smithi022:check power output: Chassis Power is on 2024-03-21T12:57:07.722 DEBUG:teuthology.orchestra.console.smithi189:power off output: Chassis Power Control: Down/Off 2024-03-21T12:57:07.723 DEBUG:teuthology.orchestra.console.smithi189:pexpect command: ipmitool -H smithi189.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-03-21T12:57:07.745 DEBUG:teuthology.orchestra.console.smithi189:check power output: Chassis Power is on 2024-03-21T12:57:11.712 DEBUG:teuthology.orchestra.console.smithi022:pexpect command: ipmitool -H smithi022.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-03-21T12:57:11.746 DEBUG:teuthology.orchestra.console.smithi189:pexpect command: ipmitool -H smithi189.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-03-21T12:57:11.822 DEBUG:teuthology.orchestra.console.smithi022:check power output: Chassis Power is on 2024-03-21T12:57:11.855 DEBUG:teuthology.orchestra.console.smithi189:check power output: Chassis Power is on 2024-03-21T12:57:15.826 DEBUG:teuthology.orchestra.console.smithi022:pexpect command: ipmitool -H smithi022.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-03-21T12:57:15.856 DEBUG:teuthology.orchestra.console.smithi189:pexpect command: ipmitool -H smithi189.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-03-21T12:57:15.935 DEBUG:teuthology.orchestra.console.smithi022:check power output: Chassis Power is off 2024-03-21T12:57:15.967 DEBUG:teuthology.orchestra.console.smithi189:check power output: Chassis Power is off 2024-03-21T12:57:16.036 INFO:teuthology.orchestra.console.smithi022:Power off completed 2024-03-21T12:57:16.068 INFO:teuthology.orchestra.console.smithi189:Power off completed 2024-03-21T12:57:16.172 INFO:teuthology.run:Summary data: description: orch:cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} duration: 2081.7620186805725 failure_reason: '"2024-03-21T12:32:21.542288+0000 mon.smithi022 (mon.0) 495 : cluster [WRN] Health check failed: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded (PG_DEGRADED)" in cluster log' owner: scheduled_gabrioux@teuthology success: false 2024-03-21T12:57:16.172 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-03-21T12:57:16.256 INFO:teuthology.run:FAIL