2024-09-20T22:02:08.221 INFO:root:teuthology version: 1.2.2.dev2+g3752d38 2024-09-20T22:02:08.239 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-20T22:02:08.354 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2024-09-20_21:08:02-orch-squid-distro-default-smithi/7912993 branch: squid description: orch/cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} email: ceph-qa@ceph.com first_in_suite: false job_id: '7912993' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi name: teuthology-2024-09-20_21:08:02-orch-squid-distro-default-smithi no_nested_subset: false openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 9.stream overrides: admin_socket: branch: squid ceph: conf: mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - CEPHADM_REFRESH_FAILED log-only-match: - CEPHADM_ sha1: 9e2ac70bb7aea67d34f0d50002344abbe7803736 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: 9e2ac70bb7aea67d34f0d50002344abbe7803736 selinux: allowlist: - scontext=system_u:system_r:logrotate_t:s0 - scontext=system_u:system_r:getty_t:s0 workunit: branch: squid sha1: 4e45e6b490e6cbe082cdf155fea780438ec53fc7 owner: scheduled_teuthology@teuthology priority: 100 repo: https://git.ceph.com/ceph.git roles: - - host.a - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - - host.b - osd.4 - osd.5 - osd.6 - osd.7 seed: 9344 sha1: 9e2ac70bb7aea67d34f0d50002344abbe7803736 sleep_before_teardown: 0 subset: 56/64 suite: orch suite_branch: squid suite_path: /home/teuthworker/src/git.ceph.com_ceph_4e45e6b490e6cbe082cdf155fea780438ec53fc7/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph.git suite_sha1: 4e45e6b490e6cbe082cdf155fea780438ec53fc7 targets: smithi037.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBCrAzclMW6DAGmN50h55szva4IBgXaxuQkygWFvGcmua2Qm8/NSrspXhZZE0ntKxAjU2fjQBsUEJI0KQvOjANtc= smithi194.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBH1LlJ15QFWuwb9VKyKwA5vPGODFfDoFbcBe9yv+AkQjxx2o2l+3RqY5A9y5T9odJyeTWE7Qt+HmmnOyAmvI7+A= tasks: - cephadm: cephadm_branch: v17.2.0 cephadm_git_url: https://github.com/ceph/ceph image: quay.io/ceph/ceph:v17.2.0 roleless: true - cephadm.shell: host.a: - ceph orch status - ceph orch ps - ceph orch ls - ceph orch host ls - ceph orch device ls - vip.exec: all-hosts: - systemctl stop nfs-server - cephadm.shell: host.a: - ceph fs volume create foofs - cephadm.wait_for_service: service: mds.foofs - cephadm.shell: host.a: - ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2 - ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake - while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done - vip.exec: host.a: - mkdir /mnt/foo - while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done - echo test > /mnt/foo/testfile - sync - parallel: - upgrade-tasks - workload-tasks - vip.exec: host.a: - umount /mnt/foo - cephadm.shell: host.a: - ceph nfs cluster ls | grep foo - ceph nfs export ls foo --detailed - rados -p .nfs --all ls - - cephadm.shell: host.a: - 'set -ex [[ `ceph config get mgr mgr/cephadm/migration_current` -gt 2 ]] ' teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a timestamp: 2024-09-20_21:08:02 tube: smithi upgrade-tasks: sequential: - cephadm.shell: env: - sha1 host.a: - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph config set global log_to_journald false --force - ceph mgr module enable nfs --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - cephadm.shell: env: - sha1 host.a: - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done - ceph orch ps - ceph versions - echo "wait for servicemap items w/ changing names to refresh" - sleep 60 - ceph orch ps - ceph orch upgrade status - ceph health detail - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 - cephadm.wait_for_service: service: nfs.foo user: teuthology verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2902 workload-tasks: sequential: - exec: host.a: - cd /mnt/foo && dbench 5 -t 600 || true - umount /mnt/foo - while ! mount -t nfs $(hostname):/fake /mnt/foo ; do sleep 5 ; done - cd /mnt/foo && dbench 5 -t 5 2024-09-20T22:02:08.355 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_4e45e6b490e6cbe082cdf155fea780438ec53fc7/qa; will attempt to use it 2024-09-20T22:02:08.355 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_4e45e6b490e6cbe082cdf155fea780438ec53fc7/qa/tasks 2024-09-20T22:02:08.355 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-09-20T22:02:08.357 INFO:teuthology.task.internal:Checking packages... 2024-09-20T22:02:08.377 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash '9e2ac70bb7aea67d34f0d50002344abbe7803736' 2024-09-20T22:02:08.377 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-20T22:02:08.377 INFO:teuthology.packaging:ref: None 2024-09-20T22:02:08.377 INFO:teuthology.packaging:tag: None 2024-09-20T22:02:08.377 INFO:teuthology.packaging:branch: squid 2024-09-20T22:02:08.377 INFO:teuthology.packaging:sha1: 9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-20T22:02:08.377 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=squid 2024-09-20T22:02:08.588 INFO:teuthology.task.internal:Found packages for ceph version 19.1.1-369.ged8ee0f2 2024-09-20T22:02:08.590 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-09-20T22:02:08.595 INFO:teuthology.task.internal:no buildpackages task found 2024-09-20T22:02:08.596 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-09-20T22:02:08.620 INFO:teuthology.task.internal:Saving configuration 2024-09-20T22:02:08.634 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-09-20T22:02:08.641 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-09-20T22:02:08.667 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi037.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-09-20_21:08:02-orch-squid-distro-default-smithi/7912993', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-20 21:57:19.995631', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBCrAzclMW6DAGmN50h55szva4IBgXaxuQkygWFvGcmua2Qm8/NSrspXhZZE0ntKxAjU2fjQBsUEJI0KQvOjANtc='} 2024-09-20T22:02:08.690 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi194.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-09-20_21:08:02-orch-squid-distro-default-smithi/7912993', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-20 21:57:19.993741', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBH1LlJ15QFWuwb9VKyKwA5vPGODFfDoFbcBe9yv+AkQjxx2o2l+3RqY5A9y5T9odJyeTWE7Qt+HmmnOyAmvI7+A='} 2024-09-20T22:02:08.691 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-09-20T22:02:08.700 INFO:teuthology.task.internal:roles: ubuntu@smithi037.front.sepia.ceph.com - ['host.a', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0'] 2024-09-20T22:02:08.700 INFO:teuthology.task.internal:roles: ubuntu@smithi194.front.sepia.ceph.com - ['host.b', 'osd.4', 'osd.5', 'osd.6', 'osd.7'] 2024-09-20T22:02:08.700 INFO:teuthology.run_tasks:Running task console_log... 2024-09-20T22:02:08.760 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f2b2892c160>, signals=[15]) 2024-09-20T22:02:08.760 INFO:teuthology.run_tasks:Running task internal.connect... 2024-09-20T22:02:08.766 INFO:teuthology.task.internal:Opening connections... 2024-09-20T22:02:08.766 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi037.front.sepia.ceph.com 2024-09-20T22:02:08.767 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi037.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-20T22:02:08.841 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi194.front.sepia.ceph.com 2024-09-20T22:02:08.842 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi194.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-20T22:02:08.915 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-09-20T22:02:08.922 DEBUG:teuthology.orchestra.run.smithi037:> uname -m 2024-09-20T22:02:08.940 INFO:teuthology.orchestra.run.smithi037.stdout:x86_64 2024-09-20T22:02:08.940 DEBUG:teuthology.orchestra.run.smithi037:> cat /etc/os-release 2024-09-20T22:02:08.997 INFO:teuthology.orchestra.run.smithi037.stdout:NAME="CentOS Stream" 2024-09-20T22:02:08.997 INFO:teuthology.orchestra.run.smithi037.stdout:VERSION="9" 2024-09-20T22:02:08.997 INFO:teuthology.orchestra.run.smithi037.stdout:ID="centos" 2024-09-20T22:02:08.997 INFO:teuthology.orchestra.run.smithi037.stdout:ID_LIKE="rhel fedora" 2024-09-20T22:02:08.997 INFO:teuthology.orchestra.run.smithi037.stdout:VERSION_ID="9" 2024-09-20T22:02:08.997 INFO:teuthology.orchestra.run.smithi037.stdout:PLATFORM_ID="platform:el9" 2024-09-20T22:02:08.997 INFO:teuthology.orchestra.run.smithi037.stdout:PRETTY_NAME="CentOS Stream 9" 2024-09-20T22:02:08.997 INFO:teuthology.orchestra.run.smithi037.stdout:ANSI_COLOR="0;31" 2024-09-20T22:02:08.997 INFO:teuthology.orchestra.run.smithi037.stdout:LOGO="fedora-logo-icon" 2024-09-20T22:02:08.997 INFO:teuthology.orchestra.run.smithi037.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-09-20T22:02:08.998 INFO:teuthology.orchestra.run.smithi037.stdout:HOME_URL="https://centos.org/" 2024-09-20T22:02:08.998 INFO:teuthology.orchestra.run.smithi037.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-09-20T22:02:08.998 INFO:teuthology.orchestra.run.smithi037.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-09-20T22:02:08.998 INFO:teuthology.orchestra.run.smithi037.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-09-20T22:02:08.998 INFO:teuthology.lock.ops:Updating smithi037.front.sepia.ceph.com on lock server 2024-09-20T22:02:09.027 DEBUG:teuthology.orchestra.run.smithi194:> uname -m 2024-09-20T22:02:09.045 INFO:teuthology.orchestra.run.smithi194.stdout:x86_64 2024-09-20T22:02:09.046 DEBUG:teuthology.orchestra.run.smithi194:> cat /etc/os-release 2024-09-20T22:02:09.104 INFO:teuthology.orchestra.run.smithi194.stdout:NAME="CentOS Stream" 2024-09-20T22:02:09.104 INFO:teuthology.orchestra.run.smithi194.stdout:VERSION="9" 2024-09-20T22:02:09.104 INFO:teuthology.orchestra.run.smithi194.stdout:ID="centos" 2024-09-20T22:02:09.104 INFO:teuthology.orchestra.run.smithi194.stdout:ID_LIKE="rhel fedora" 2024-09-20T22:02:09.104 INFO:teuthology.orchestra.run.smithi194.stdout:VERSION_ID="9" 2024-09-20T22:02:09.105 INFO:teuthology.orchestra.run.smithi194.stdout:PLATFORM_ID="platform:el9" 2024-09-20T22:02:09.105 INFO:teuthology.orchestra.run.smithi194.stdout:PRETTY_NAME="CentOS Stream 9" 2024-09-20T22:02:09.105 INFO:teuthology.orchestra.run.smithi194.stdout:ANSI_COLOR="0;31" 2024-09-20T22:02:09.105 INFO:teuthology.orchestra.run.smithi194.stdout:LOGO="fedora-logo-icon" 2024-09-20T22:02:09.105 INFO:teuthology.orchestra.run.smithi194.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-09-20T22:02:09.105 INFO:teuthology.orchestra.run.smithi194.stdout:HOME_URL="https://centos.org/" 2024-09-20T22:02:09.105 INFO:teuthology.orchestra.run.smithi194.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-09-20T22:02:09.105 INFO:teuthology.orchestra.run.smithi194.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-09-20T22:02:09.105 INFO:teuthology.orchestra.run.smithi194.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-09-20T22:02:09.105 INFO:teuthology.lock.ops:Updating smithi194.front.sepia.ceph.com on lock server 2024-09-20T22:02:09.127 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-09-20T22:02:09.136 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-09-20T22:02:09.216 INFO:teuthology.task.internal:Checking for old test directory... 2024-09-20T22:02:09.217 DEBUG:teuthology.orchestra.run.smithi037:> test '!' -e /home/ubuntu/cephtest 2024-09-20T22:02:09.219 DEBUG:teuthology.orchestra.run.smithi194:> test '!' -e /home/ubuntu/cephtest 2024-09-20T22:02:09.236 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-09-20T22:02:09.242 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-09-20T22:02:09.243 DEBUG:teuthology.orchestra.run.smithi037:> test -z $(ls -A /var/lib/ceph) 2024-09-20T22:02:09.277 DEBUG:teuthology.orchestra.run.smithi194:> test -z $(ls -A /var/lib/ceph) 2024-09-20T22:02:09.385 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-09-20T22:02:09.447 INFO:teuthology.run_tasks:Running task kernel... 2024-09-20T22:02:09.533 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-09-20T22:02:09.533 INFO:teuthology.task.kernel:config {'host.a': {'kdb': 1, 'sha1': 'distro'}, 'host.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-09-20T22:02:09.534 DEBUG:teuthology.orchestra.run.smithi037:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-20T22:02:09.534 DEBUG:teuthology.orchestra.run.smithi194:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-20T22:02:09.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-20T22:02:09.551 DEBUG:teuthology.orchestra.run.smithi037:> uname -r 2024-09-20T22:02:09.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-20T22:02:09.552 DEBUG:teuthology.orchestra.run.smithi194:> uname -r 2024-09-20T22:02:09.609 INFO:teuthology.orchestra.run.smithi037.stdout:5.14.0-503.el9.x86_64 2024-09-20T22:02:09.609 INFO:teuthology.task.kernel:Running kernel on smithi037: 5.14.0-503.el9.x86_64 2024-09-20T22:02:09.609 DEBUG:teuthology.orchestra.run.smithi037:> sudo yum install -y kernel 2024-09-20T22:02:09.610 INFO:teuthology.orchestra.run.smithi194.stdout:5.14.0-503.el9.x86_64 2024-09-20T22:02:09.610 INFO:teuthology.task.kernel:Running kernel on smithi194: 5.14.0-503.el9.x86_64 2024-09-20T22:02:09.610 DEBUG:teuthology.orchestra.run.smithi194:> sudo yum install -y kernel 2024-09-20T22:02:11.802 INFO:teuthology.orchestra.run.smithi194.stdout:CentOS Stream 9 - BaseOS 16 MB/s | 8.3 MB 00:00 2024-09-20T22:02:11.924 INFO:teuthology.orchestra.run.smithi037.stdout:CentOS Stream 9 - BaseOS 13 MB/s | 8.3 MB 00:00 2024-09-20T22:02:16.641 INFO:teuthology.orchestra.run.smithi037.stdout:CentOS Stream 9 - AppStream 5.7 MB/s | 20 MB 00:03 2024-09-20T22:02:20.669 INFO:teuthology.orchestra.run.smithi194.stdout:CentOS Stream 9 - AppStream 2.6 MB/s | 20 MB 00:07 2024-09-20T22:02:20.958 INFO:teuthology.orchestra.run.smithi037.stdout:CentOS Stream 9 - CRB 17 MB/s | 6.5 MB 00:00 2024-09-20T22:02:23.810 INFO:teuthology.orchestra.run.smithi037.stdout:CentOS Stream 9 - Extras packages 12 kB/s | 19 kB 00:01 2024-09-20T22:02:24.889 INFO:teuthology.orchestra.run.smithi037.stdout:Extra Packages for Enterprise Linux 24 MB/s | 23 MB 00:00 2024-09-20T22:02:24.911 INFO:teuthology.orchestra.run.smithi194.stdout:CentOS Stream 9 - CRB 14 MB/s | 6.5 MB 00:00 2024-09-20T22:02:26.838 INFO:teuthology.orchestra.run.smithi194.stdout:CentOS Stream 9 - Extras packages 27 kB/s | 19 kB 00:00 2024-09-20T22:02:27.990 INFO:teuthology.orchestra.run.smithi194.stdout:Extra Packages for Enterprise Linux 22 MB/s | 23 MB 00:01 2024-09-20T22:02:30.553 INFO:teuthology.orchestra.run.smithi037.stdout:lab-extras 30 kB/s | 1.7 kB 00:00 2024-09-20T22:02:32.568 INFO:teuthology.orchestra.run.smithi037.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-09-20T22:02:32.568 INFO:teuthology.orchestra.run.smithi037.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-09-20T22:02:32.664 INFO:teuthology.orchestra.run.smithi037.stdout:Dependencies resolved. 2024-09-20T22:02:32.675 INFO:teuthology.orchestra.run.smithi037.stdout:================================================================================ 2024-09-20T22:02:32.676 INFO:teuthology.orchestra.run.smithi037.stdout: Package Architecture Version Repository Size 2024-09-20T22:02:32.676 INFO:teuthology.orchestra.run.smithi037.stdout:================================================================================ 2024-09-20T22:02:32.676 INFO:teuthology.orchestra.run.smithi037.stdout:Installing: 2024-09-20T22:02:32.676 INFO:teuthology.orchestra.run.smithi037.stdout: kernel x86_64 5.14.0-508.el9 baseos 36 k 2024-09-20T22:02:32.676 INFO:teuthology.orchestra.run.smithi037.stdout:Installing dependencies: 2024-09-20T22:02:32.676 INFO:teuthology.orchestra.run.smithi037.stdout: kernel-core x86_64 5.14.0-508.el9 baseos 16 M 2024-09-20T22:02:32.676 INFO:teuthology.orchestra.run.smithi037.stdout: kernel-modules x86_64 5.14.0-508.el9 baseos 35 M 2024-09-20T22:02:32.676 INFO:teuthology.orchestra.run.smithi037.stdout: kernel-modules-core x86_64 5.14.0-508.el9 baseos 29 M 2024-09-20T22:02:32.676 INFO:teuthology.orchestra.run.smithi037.stdout: 2024-09-20T22:02:32.676 INFO:teuthology.orchestra.run.smithi037.stdout:Transaction Summary 2024-09-20T22:02:32.676 INFO:teuthology.orchestra.run.smithi037.stdout:================================================================================ 2024-09-20T22:02:32.676 INFO:teuthology.orchestra.run.smithi037.stdout:Install 4 Packages 2024-09-20T22:02:32.676 INFO:teuthology.orchestra.run.smithi037.stdout: 2024-09-20T22:02:32.677 INFO:teuthology.orchestra.run.smithi037.stdout:Total download size: 79 M 2024-09-20T22:02:32.677 INFO:teuthology.orchestra.run.smithi037.stdout:Installed size: 126 M 2024-09-20T22:02:32.677 INFO:teuthology.orchestra.run.smithi037.stdout:Downloading Packages: 2024-09-20T22:02:32.838 INFO:teuthology.orchestra.run.smithi037.stdout:(1/4): kernel-5.14.0-508.el9.x86_64.rpm 318 kB/s | 36 kB 00:00 2024-09-20T22:02:33.429 INFO:teuthology.orchestra.run.smithi194.stdout:lab-extras 29 kB/s | 1.7 kB 00:00 2024-09-20T22:02:35.328 INFO:teuthology.orchestra.run.smithi194.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-09-20T22:02:35.328 INFO:teuthology.orchestra.run.smithi194.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-09-20T22:02:35.427 INFO:teuthology.orchestra.run.smithi194.stdout:Dependencies resolved. 2024-09-20T22:02:35.434 INFO:teuthology.orchestra.run.smithi194.stdout:================================================================================ 2024-09-20T22:02:35.434 INFO:teuthology.orchestra.run.smithi194.stdout: Package Architecture Version Repository Size 2024-09-20T22:02:35.434 INFO:teuthology.orchestra.run.smithi194.stdout:================================================================================ 2024-09-20T22:02:35.434 INFO:teuthology.orchestra.run.smithi194.stdout:Installing: 2024-09-20T22:02:35.434 INFO:teuthology.orchestra.run.smithi194.stdout: kernel x86_64 5.14.0-508.el9 baseos 36 k 2024-09-20T22:02:35.434 INFO:teuthology.orchestra.run.smithi194.stdout:Installing dependencies: 2024-09-20T22:02:35.434 INFO:teuthology.orchestra.run.smithi194.stdout: kernel-core x86_64 5.14.0-508.el9 baseos 16 M 2024-09-20T22:02:35.434 INFO:teuthology.orchestra.run.smithi194.stdout: kernel-modules x86_64 5.14.0-508.el9 baseos 35 M 2024-09-20T22:02:35.435 INFO:teuthology.orchestra.run.smithi194.stdout: kernel-modules-core x86_64 5.14.0-508.el9 baseos 29 M 2024-09-20T22:02:35.435 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-09-20T22:02:35.435 INFO:teuthology.orchestra.run.smithi194.stdout:Transaction Summary 2024-09-20T22:02:35.435 INFO:teuthology.orchestra.run.smithi194.stdout:================================================================================ 2024-09-20T22:02:35.435 INFO:teuthology.orchestra.run.smithi194.stdout:Install 4 Packages 2024-09-20T22:02:35.435 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-09-20T22:02:35.435 INFO:teuthology.orchestra.run.smithi194.stdout:Total download size: 79 M 2024-09-20T22:02:35.435 INFO:teuthology.orchestra.run.smithi194.stdout:Installed size: 126 M 2024-09-20T22:02:35.436 INFO:teuthology.orchestra.run.smithi194.stdout:Downloading Packages: 2024-09-20T22:02:35.831 INFO:teuthology.orchestra.run.smithi194.stdout:(1/4): kernel-5.14.0-508.el9.x86_64.rpm 311 kB/s | 36 kB 00:00 2024-09-20T22:02:37.379 INFO:teuthology.orchestra.run.smithi037.stdout:(2/4): kernel-modules-core-5.14.0-508.el9.x86_6 6.3 MB/s | 29 MB 00:04 2024-09-20T22:02:39.005 INFO:teuthology.orchestra.run.smithi194.stdout:(2/4): kernel-core-5.14.0-508.el9.x86_64.rpm 4.8 MB/s | 16 MB 00:03 2024-09-20T22:02:39.871 INFO:teuthology.orchestra.run.smithi037.stdout:(3/4): kernel-core-5.14.0-508.el9.x86_64.rpm 2.2 MB/s | 16 MB 00:07 2024-09-20T22:02:40.014 INFO:teuthology.orchestra.run.smithi194.stdout:(3/4): kernel-modules-core-5.14.0-508.el9.x86_6 6.8 MB/s | 29 MB 00:04 2024-09-20T22:02:40.831 INFO:teuthology.orchestra.run.smithi194.stdout:(4/4): kernel-modules-5.14.0-508.el9.x86_64.rpm 6.8 MB/s | 35 MB 00:05 2024-09-20T22:02:40.831 INFO:teuthology.orchestra.run.smithi194.stdout:-------------------------------------------------------------------------------- 2024-09-20T22:02:40.831 INFO:teuthology.orchestra.run.smithi194.stdout:Total 15 MB/s | 79 MB 00:05 2024-09-20T22:02:41.515 INFO:teuthology.orchestra.run.smithi194.stdout:Running transaction check 2024-09-20T22:02:41.855 INFO:teuthology.orchestra.run.smithi194.stdout:Transaction check succeeded. 2024-09-20T22:02:41.855 INFO:teuthology.orchestra.run.smithi194.stdout:Running transaction test 2024-09-20T22:02:42.804 INFO:teuthology.orchestra.run.smithi194.stdout:Transaction test succeeded. 2024-09-20T22:02:42.805 INFO:teuthology.orchestra.run.smithi194.stdout:Running transaction 2024-09-20T22:02:43.596 INFO:teuthology.orchestra.run.smithi037.stdout:(4/4): kernel-modules-5.14.0-508.el9.x86_64.rpm 3.2 MB/s | 35 MB 00:10 2024-09-20T22:02:43.596 INFO:teuthology.orchestra.run.smithi037.stdout:-------------------------------------------------------------------------------- 2024-09-20T22:02:43.596 INFO:teuthology.orchestra.run.smithi037.stdout:Total 7.2 MB/s | 79 MB 00:10 2024-09-20T22:02:44.306 INFO:teuthology.orchestra.run.smithi037.stdout:Running transaction check 2024-09-20T22:02:44.677 INFO:teuthology.orchestra.run.smithi037.stdout:Transaction check succeeded. 2024-09-20T22:02:44.677 INFO:teuthology.orchestra.run.smithi037.stdout:Running transaction test 2024-09-20T22:02:45.230 INFO:teuthology.orchestra.run.smithi194.stdout: Preparing : 1/1 2024-09-20T22:02:45.640 INFO:teuthology.orchestra.run.smithi037.stdout:Transaction test succeeded. 2024-09-20T22:02:45.640 INFO:teuthology.orchestra.run.smithi037.stdout:Running transaction 2024-09-20T22:02:45.965 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : kernel-modules-core-5.14.0-508.el9.x86_64 1/4 2024-09-20T22:02:46.055 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-20T22:02:48.230 INFO:teuthology.orchestra.run.smithi037.stdout: Preparing : 1/1 2024-09-20T22:02:48.836 INFO:teuthology.orchestra.run.smithi194.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-20T22:02:48.900 INFO:teuthology.orchestra.run.smithi037.stdout: Installing : kernel-modules-core-5.14.0-508.el9.x86_64 1/4 2024-09-20T22:02:48.958 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-20T22:02:48.980 INFO:teuthology.orchestra.run.smithi037.stdout: Installing : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-20T22:02:51.786 INFO:teuthology.orchestra.run.smithi037.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-20T22:02:51.926 INFO:teuthology.orchestra.run.smithi037.stdout: Installing : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-20T22:02:53.781 INFO:teuthology.orchestra.run.smithi194.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-20T22:02:54.117 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : kernel-5.14.0-508.el9.x86_64 4/4 2024-09-20T22:02:57.053 INFO:teuthology.orchestra.run.smithi037.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-20T22:02:57.093 INFO:teuthology.orchestra.run.smithi037.stdout: Installing : kernel-5.14.0-508.el9.x86_64 4/4 2024-09-20T22:02:58.944 INFO:teuthology.orchestra.run.smithi194.stdout: Running scriptlet: kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-20T22:03:02.230 INFO:teuthology.orchestra.run.smithi037.stdout: Running scriptlet: kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-20T22:03:36.842 INFO:teuthology.orchestra.run.smithi194.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 4/4 2024-09-20T22:03:36.882 INFO:teuthology.orchestra.run.smithi194.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 4/4 2024-09-20T22:03:37.154 INFO:teuthology.orchestra.run.smithi194.stdout: Running scriptlet: kernel-5.14.0-508.el9.x86_64 4/4 2024-09-20T22:03:37.154 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : kernel-5.14.0-508.el9.x86_64 1/4 2024-09-20T22:03:37.155 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-20T22:03:37.155 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-20T22:03:37.975 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-20T22:03:37.975 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-09-20T22:03:37.975 INFO:teuthology.orchestra.run.smithi194.stdout:Installed: 2024-09-20T22:03:37.975 INFO:teuthology.orchestra.run.smithi194.stdout: kernel-5.14.0-508.el9.x86_64 2024-09-20T22:03:37.975 INFO:teuthology.orchestra.run.smithi194.stdout: kernel-core-5.14.0-508.el9.x86_64 2024-09-20T22:03:37.975 INFO:teuthology.orchestra.run.smithi194.stdout: kernel-modules-5.14.0-508.el9.x86_64 2024-09-20T22:03:37.975 INFO:teuthology.orchestra.run.smithi194.stdout: kernel-modules-core-5.14.0-508.el9.x86_64 2024-09-20T22:03:37.975 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-09-20T22:03:37.975 INFO:teuthology.orchestra.run.smithi194.stdout:Complete! 2024-09-20T22:03:38.325 DEBUG:teuthology.orchestra.run.smithi194:> rpm -q kernel | sort -rV | head -n 1 2024-09-20T22:03:38.405 INFO:teuthology.orchestra.run.smithi194.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-20T22:03:38.405 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-20T22:03:38.405 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-503.el9.x86_64 Expected: 5.14.0-508.el9.x86_64 2024-09-20T22:03:38.405 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-20T22:03:38.405 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-20T22:03:38.405 INFO:teuthology.task.kernel:Installing distro kernel on host.b... 2024-09-20T22:03:38.405 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi194.front.sepia.ceph.com, path=None, version=distro) 2024-09-20T22:03:38.406 DEBUG:teuthology.orchestra.run.smithi194:> rpm -q kernel | sort -rV | head -n 1 2024-09-20T22:03:38.463 INFO:teuthology.orchestra.run.smithi194.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-20T22:03:38.464 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-20T22:03:38.464 DEBUG:teuthology.orchestra.run.smithi194:> sudo rpm -qi grub2-tools 2024-09-20T22:03:38.503 INFO:teuthology.orchestra.run.smithi194.stdout:Name : grub2-tools 2024-09-20T22:03:38.503 INFO:teuthology.orchestra.run.smithi194.stdout:Epoch : 1 2024-09-20T22:03:38.503 INFO:teuthology.orchestra.run.smithi194.stdout:Version : 2.06 2024-09-20T22:03:38.503 INFO:teuthology.orchestra.run.smithi194.stdout:Release : 82.el9 2024-09-20T22:03:38.503 INFO:teuthology.orchestra.run.smithi194.stdout:Architecture: x86_64 2024-09-20T22:03:38.503 INFO:teuthology.orchestra.run.smithi194.stdout:Install Date: Wed 28 Aug 2024 09:06:27 PM UTC 2024-09-20T22:03:38.503 INFO:teuthology.orchestra.run.smithi194.stdout:Group : Unspecified 2024-09-20T22:03:38.503 INFO:teuthology.orchestra.run.smithi194.stdout:Size : 8274670 2024-09-20T22:03:38.504 INFO:teuthology.orchestra.run.smithi194.stdout:License : GPLv3+ 2024-09-20T22:03:38.504 INFO:teuthology.orchestra.run.smithi194.stdout:Signature : RSA/SHA256, Mon 01 Jul 2024 07:46:42 AM UTC, Key ID 05b555b38483c65d 2024-09-20T22:03:38.504 INFO:teuthology.orchestra.run.smithi194.stdout:Source RPM : grub2-2.06-82.el9.src.rpm 2024-09-20T22:03:38.504 INFO:teuthology.orchestra.run.smithi194.stdout:Build Date : Fri 28 Jun 2024 09:24:44 AM UTC 2024-09-20T22:03:38.504 INFO:teuthology.orchestra.run.smithi194.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-09-20T22:03:38.504 INFO:teuthology.orchestra.run.smithi194.stdout:Packager : builder@centos.org 2024-09-20T22:03:38.504 INFO:teuthology.orchestra.run.smithi194.stdout:Vendor : CentOS 2024-09-20T22:03:38.504 INFO:teuthology.orchestra.run.smithi194.stdout:URL : http://www.gnu.org/software/grub/ 2024-09-20T22:03:38.504 INFO:teuthology.orchestra.run.smithi194.stdout:Summary : Support tools for GRUB. 2024-09-20T22:03:38.504 INFO:teuthology.orchestra.run.smithi194.stdout:Description : 2024-09-20T22:03:38.504 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-09-20T22:03:38.504 INFO:teuthology.orchestra.run.smithi194.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-09-20T22:03:38.504 INFO:teuthology.orchestra.run.smithi194.stdout:customizable bootloader with modular architecture. It supports a rich 2024-09-20T22:03:38.504 INFO:teuthology.orchestra.run.smithi194.stdout:variety of kernel formats, file systems, computer architectures and 2024-09-20T22:03:38.504 INFO:teuthology.orchestra.run.smithi194.stdout:hardware devices. 2024-09-20T22:03:38.505 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-09-20T22:03:38.505 INFO:teuthology.orchestra.run.smithi194.stdout:This subpackage provides tools for support of all platforms. 2024-09-20T22:03:38.505 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-09-20T22:03:38.505 INFO:teuthology.task.kernel:Updating grub on smithi194 to boot 5.14.0-508.el9.x86_64 2024-09-20T22:03:38.506 DEBUG:teuthology.orchestra.run.smithi194:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-09-20T22:03:39.222 INFO:teuthology.orchestra.run.smithi194.stderr:Generating grub configuration file ... 2024-09-20T22:03:40.523 INFO:teuthology.orchestra.run.smithi194.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-09-20T22:03:40.551 INFO:teuthology.orchestra.run.smithi194.stderr:done 2024-09-20T22:03:40.554 DEBUG:teuthology.orchestra.run.smithi194:> mktemp 2024-09-20T22:03:40.569 INFO:teuthology.orchestra.run.smithi194.stdout:/tmp/tmp.MMLZ2vjtaY 2024-09-20T22:03:40.569 DEBUG:teuthology.orchestra.run.smithi194:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.MMLZ2vjtaY 2024-09-20T22:03:40.646 DEBUG:teuthology.orchestra.run.smithi194:> sudo chmod 0666 /tmp/tmp.MMLZ2vjtaY 2024-09-20T22:03:40.779 DEBUG:teuthology.orchestra.remote:smithi194:/tmp/tmp.MMLZ2vjtaY is 6KB 2024-09-20T22:03:40.800 DEBUG:teuthology.orchestra.run.smithi194:> rm -fr /tmp/tmp.MMLZ2vjtaY 2024-09-20T22:03:40.815 DEBUG:teuthology.orchestra.run.smithi194:> sudo /bin/ls /boot/loader/entries || true 2024-09-20T22:03:40.877 INFO:teuthology.orchestra.run.smithi194.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-0-rescue.conf 2024-09-20T22:03:40.877 INFO:teuthology.orchestra.run.smithi194.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-239.el9.x86_64.conf 2024-09-20T22:03:40.877 INFO:teuthology.orchestra.run.smithi194.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-503.el9.x86_64.conf 2024-09-20T22:03:40.878 INFO:teuthology.orchestra.run.smithi194.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64.conf 2024-09-20T22:03:40.879 DEBUG:teuthology.orchestra.run.smithi194:> sudo grub2-set-default fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64 2024-09-20T22:03:41.032 DEBUG:teuthology.orchestra.run.smithi194:> sudo shutdown -r now 2024-09-20T22:03:41.471 INFO:teuthology.orchestra.run.smithi037.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 4/4 2024-09-20T22:03:41.513 INFO:teuthology.orchestra.run.smithi037.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 4/4 2024-09-20T22:03:41.879 INFO:teuthology.orchestra.run.smithi037.stdout: Running scriptlet: kernel-5.14.0-508.el9.x86_64 4/4 2024-09-20T22:03:41.879 INFO:teuthology.orchestra.run.smithi037.stdout: Verifying : kernel-5.14.0-508.el9.x86_64 1/4 2024-09-20T22:03:41.879 INFO:teuthology.orchestra.run.smithi037.stdout: Verifying : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-20T22:03:41.879 INFO:teuthology.orchestra.run.smithi037.stdout: Verifying : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-20T22:03:42.710 INFO:teuthology.orchestra.run.smithi037.stdout: Verifying : kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-20T22:03:42.710 INFO:teuthology.orchestra.run.smithi037.stdout: 2024-09-20T22:03:42.710 INFO:teuthology.orchestra.run.smithi037.stdout:Installed: 2024-09-20T22:03:42.710 INFO:teuthology.orchestra.run.smithi037.stdout: kernel-5.14.0-508.el9.x86_64 2024-09-20T22:03:42.710 INFO:teuthology.orchestra.run.smithi037.stdout: kernel-core-5.14.0-508.el9.x86_64 2024-09-20T22:03:42.711 INFO:teuthology.orchestra.run.smithi037.stdout: kernel-modules-5.14.0-508.el9.x86_64 2024-09-20T22:03:42.711 INFO:teuthology.orchestra.run.smithi037.stdout: kernel-modules-core-5.14.0-508.el9.x86_64 2024-09-20T22:03:42.711 INFO:teuthology.orchestra.run.smithi037.stdout: 2024-09-20T22:03:42.711 INFO:teuthology.orchestra.run.smithi037.stdout:Complete! 2024-09-20T22:03:43.088 DEBUG:teuthology.orchestra.run.smithi037:> rpm -q kernel | sort -rV | head -n 1 2024-09-20T22:03:43.174 INFO:teuthology.orchestra.run.smithi037.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-20T22:03:43.175 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-20T22:03:43.175 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-503.el9.x86_64 Expected: 5.14.0-508.el9.x86_64 2024-09-20T22:03:43.175 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-20T22:03:43.175 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-20T22:03:43.175 INFO:teuthology.task.kernel:Installing distro kernel on host.a... 2024-09-20T22:03:43.175 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi037.front.sepia.ceph.com, path=None, version=distro) 2024-09-20T22:03:43.175 DEBUG:teuthology.orchestra.run.smithi037:> rpm -q kernel | sort -rV | head -n 1 2024-09-20T22:03:43.234 INFO:teuthology.orchestra.run.smithi037.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-20T22:03:43.234 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-20T22:03:43.234 DEBUG:teuthology.orchestra.run.smithi037:> sudo rpm -qi grub2-tools 2024-09-20T22:03:43.268 INFO:teuthology.orchestra.run.smithi037.stdout:Name : grub2-tools 2024-09-20T22:03:43.268 INFO:teuthology.orchestra.run.smithi037.stdout:Epoch : 1 2024-09-20T22:03:43.268 INFO:teuthology.orchestra.run.smithi037.stdout:Version : 2.06 2024-09-20T22:03:43.268 INFO:teuthology.orchestra.run.smithi037.stdout:Release : 82.el9 2024-09-20T22:03:43.268 INFO:teuthology.orchestra.run.smithi037.stdout:Architecture: x86_64 2024-09-20T22:03:43.268 INFO:teuthology.orchestra.run.smithi037.stdout:Install Date: Wed 28 Aug 2024 09:06:27 PM UTC 2024-09-20T22:03:43.268 INFO:teuthology.orchestra.run.smithi037.stdout:Group : Unspecified 2024-09-20T22:03:43.268 INFO:teuthology.orchestra.run.smithi037.stdout:Size : 8274670 2024-09-20T22:03:43.268 INFO:teuthology.orchestra.run.smithi037.stdout:License : GPLv3+ 2024-09-20T22:03:43.268 INFO:teuthology.orchestra.run.smithi037.stdout:Signature : RSA/SHA256, Mon 01 Jul 2024 07:46:42 AM UTC, Key ID 05b555b38483c65d 2024-09-20T22:03:43.268 INFO:teuthology.orchestra.run.smithi037.stdout:Source RPM : grub2-2.06-82.el9.src.rpm 2024-09-20T22:03:43.268 INFO:teuthology.orchestra.run.smithi037.stdout:Build Date : Fri 28 Jun 2024 09:24:44 AM UTC 2024-09-20T22:03:43.269 INFO:teuthology.orchestra.run.smithi037.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-09-20T22:03:43.269 INFO:teuthology.orchestra.run.smithi037.stdout:Packager : builder@centos.org 2024-09-20T22:03:43.269 INFO:teuthology.orchestra.run.smithi037.stdout:Vendor : CentOS 2024-09-20T22:03:43.269 INFO:teuthology.orchestra.run.smithi037.stdout:URL : http://www.gnu.org/software/grub/ 2024-09-20T22:03:43.269 INFO:teuthology.orchestra.run.smithi037.stdout:Summary : Support tools for GRUB. 2024-09-20T22:03:43.269 INFO:teuthology.orchestra.run.smithi037.stdout:Description : 2024-09-20T22:03:43.269 INFO:teuthology.orchestra.run.smithi037.stdout: 2024-09-20T22:03:43.269 INFO:teuthology.orchestra.run.smithi037.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-09-20T22:03:43.269 INFO:teuthology.orchestra.run.smithi037.stdout:customizable bootloader with modular architecture. It supports a rich 2024-09-20T22:03:43.269 INFO:teuthology.orchestra.run.smithi037.stdout:variety of kernel formats, file systems, computer architectures and 2024-09-20T22:03:43.269 INFO:teuthology.orchestra.run.smithi037.stdout:hardware devices. 2024-09-20T22:03:43.269 INFO:teuthology.orchestra.run.smithi037.stdout: 2024-09-20T22:03:43.269 INFO:teuthology.orchestra.run.smithi037.stdout:This subpackage provides tools for support of all platforms. 2024-09-20T22:03:43.270 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-09-20T22:03:43.270 INFO:teuthology.task.kernel:Updating grub on smithi037 to boot 5.14.0-508.el9.x86_64 2024-09-20T22:03:43.270 DEBUG:teuthology.orchestra.run.smithi037:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-09-20T22:03:43.902 INFO:teuthology.orchestra.run.smithi037.stderr:Generating grub configuration file ... 2024-09-20T22:03:45.329 INFO:teuthology.orchestra.run.smithi037.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-09-20T22:03:45.357 INFO:teuthology.orchestra.run.smithi037.stderr:done 2024-09-20T22:03:45.359 DEBUG:teuthology.orchestra.run.smithi037:> mktemp 2024-09-20T22:03:45.375 INFO:teuthology.orchestra.run.smithi037.stdout:/tmp/tmp.nJaTo92sJq 2024-09-20T22:03:45.375 DEBUG:teuthology.orchestra.run.smithi037:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.nJaTo92sJq 2024-09-20T22:03:45.441 DEBUG:teuthology.orchestra.run.smithi037:> sudo chmod 0666 /tmp/tmp.nJaTo92sJq 2024-09-20T22:03:45.575 DEBUG:teuthology.orchestra.remote:smithi037:/tmp/tmp.nJaTo92sJq is 6KB 2024-09-20T22:03:45.623 DEBUG:teuthology.orchestra.run.smithi037:> rm -fr /tmp/tmp.nJaTo92sJq 2024-09-20T22:03:45.638 DEBUG:teuthology.orchestra.run.smithi037:> sudo /bin/ls /boot/loader/entries || true 2024-09-20T22:03:45.700 INFO:teuthology.orchestra.run.smithi037.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-0-rescue.conf 2024-09-20T22:03:45.700 INFO:teuthology.orchestra.run.smithi037.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-239.el9.x86_64.conf 2024-09-20T22:03:45.700 INFO:teuthology.orchestra.run.smithi037.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-503.el9.x86_64.conf 2024-09-20T22:03:45.700 INFO:teuthology.orchestra.run.smithi037.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64.conf 2024-09-20T22:03:45.701 DEBUG:teuthology.orchestra.run.smithi037:> sudo grub2-set-default fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64 2024-09-20T22:03:45.852 DEBUG:teuthology.orchestra.run.smithi037:> sudo shutdown -r now 2024-09-20T22:04:11.061 INFO:teuthology.task.kernel:Checking client host.b for new kernel version... 2024-09-20T22:04:11.061 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi194.front.sepia.ceph.com' 2024-09-20T22:04:11.062 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi194.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-20T22:04:15.859 INFO:teuthology.task.kernel:Checking client host.a for new kernel version... 2024-09-20T22:04:15.860 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi037.front.sepia.ceph.com' 2024-09-20T22:04:15.860 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi037.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-20T22:05:11.119 DEBUG:teuthology.orchestra.remote:timed out 2024-09-20T22:05:15.865 DEBUG:teuthology.orchestra.remote:timed out 2024-09-20T22:05:20.122 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi194.front.sepia.ceph.com' 2024-09-20T22:05:20.123 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi194.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-20T22:05:20.495 DEBUG:teuthology.orchestra.run.smithi194:> true 2024-09-20T22:05:20.846 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi194.front.sepia.ceph.com' 2024-09-20T22:05:20.846 INFO:teuthology.task.kernel:Checking kernel version of host.b, want "5.14.0-508.el9.x86_64"... 2024-09-20T22:05:20.846 DEBUG:teuthology.orchestra.run.smithi194:> uname -r 2024-09-20T22:05:20.861 INFO:teuthology.orchestra.run.smithi194.stdout:5.14.0-508.el9.x86_64 2024-09-20T22:05:20.862 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-508.el9.x86_64 vs 5.14.0-508.el9.x86_64 2024-09-20T22:05:20.862 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-20T22:05:20.862 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-09-20T22:05:21.863 INFO:teuthology.task.kernel:Enabling kdb on host.b... 2024-09-20T22:05:21.863 DEBUG:teuthology.orchestra.run.smithi194:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-20T22:05:21.973 INFO:teuthology.orchestra.run.smithi194.stdout:ttyS1 2024-09-20T22:05:21.991 DEBUG:teuthology.parallel:result is None 2024-09-20T22:05:24.866 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi037.front.sepia.ceph.com' 2024-09-20T22:05:24.866 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi037.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-20T22:05:25.221 DEBUG:teuthology.orchestra.run.smithi037:> true 2024-09-20T22:05:25.660 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi037.front.sepia.ceph.com' 2024-09-20T22:05:25.660 INFO:teuthology.task.kernel:Checking kernel version of host.a, want "5.14.0-508.el9.x86_64"... 2024-09-20T22:05:25.660 DEBUG:teuthology.orchestra.run.smithi037:> uname -r 2024-09-20T22:05:25.675 INFO:teuthology.orchestra.run.smithi037.stdout:5.14.0-508.el9.x86_64 2024-09-20T22:05:25.675 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-508.el9.x86_64 vs 5.14.0-508.el9.x86_64 2024-09-20T22:05:25.675 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-20T22:05:25.675 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-09-20T22:05:26.676 INFO:teuthology.task.kernel:Enabling kdb on host.a... 2024-09-20T22:05:26.676 DEBUG:teuthology.orchestra.run.smithi037:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-20T22:05:26.748 INFO:teuthology.orchestra.run.smithi037.stdout:ttyS1 2024-09-20T22:05:26.773 DEBUG:teuthology.parallel:result is None 2024-09-20T22:05:26.773 INFO:teuthology.run_tasks:Running task internal.base... 2024-09-20T22:05:26.780 INFO:teuthology.task.internal:Creating test directory... 2024-09-20T22:05:26.780 DEBUG:teuthology.orchestra.run.smithi037:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-20T22:05:26.782 DEBUG:teuthology.orchestra.run.smithi194:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-20T22:05:26.802 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-09-20T22:05:26.808 INFO:teuthology.run_tasks:Running task internal.archive... 2024-09-20T22:05:26.818 INFO:teuthology.task.internal:Creating archive directory... 2024-09-20T22:05:26.818 DEBUG:teuthology.orchestra.run.smithi037:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-20T22:05:26.838 DEBUG:teuthology.orchestra.run.smithi194:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-20T22:05:26.887 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-09-20T22:05:26.892 INFO:teuthology.task.internal:Enabling coredump saving... 2024-09-20T22:05:26.892 DEBUG:teuthology.orchestra.run.smithi037:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-20T22:05:26.906 DEBUG:teuthology.orchestra.run.smithi194:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-20T22:05:26.941 INFO:teuthology.orchestra.run.smithi037.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-20T22:05:26.952 INFO:teuthology.orchestra.run.smithi037.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-20T22:05:26.953 INFO:teuthology.orchestra.run.smithi194.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-20T22:05:26.965 INFO:teuthology.orchestra.run.smithi194.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-20T22:05:26.966 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-09-20T22:05:26.973 INFO:teuthology.task.internal:Configuring sudo... 2024-09-20T22:05:26.973 DEBUG:teuthology.orchestra.run.smithi037:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-20T22:05:26.996 DEBUG:teuthology.orchestra.run.smithi194:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-20T22:05:27.035 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-09-20T22:05:27.043 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-09-20T22:05:27.043 DEBUG:teuthology.orchestra.run.smithi037:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-20T22:05:27.061 DEBUG:teuthology.orchestra.run.smithi194:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-20T22:05:27.092 DEBUG:teuthology.orchestra.run.smithi037:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-20T22:05:27.160 DEBUG:teuthology.orchestra.run.smithi037:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-20T22:05:27.214 DEBUG:teuthology.orchestra.run.smithi037:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-20T22:05:27.296 DEBUG:teuthology.orchestra.run.smithi037:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-20T22:05:27.336 DEBUG:teuthology.orchestra.run.smithi037:> set -ex 2024-09-20T22:05:27.336 DEBUG:teuthology.orchestra.run.smithi037:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-20T22:05:27.402 DEBUG:teuthology.orchestra.run.smithi194:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-20T22:05:27.448 DEBUG:teuthology.orchestra.run.smithi194:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-20T22:05:27.501 DEBUG:teuthology.orchestra.run.smithi194:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-20T22:05:27.584 DEBUG:teuthology.orchestra.run.smithi194:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-20T22:05:27.628 DEBUG:teuthology.orchestra.run.smithi194:> set -ex 2024-09-20T22:05:27.628 DEBUG:teuthology.orchestra.run.smithi194:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-20T22:05:27.696 DEBUG:teuthology.orchestra.run.smithi037:> sudo service rsyslog restart 2024-09-20T22:05:27.698 DEBUG:teuthology.orchestra.run.smithi194:> sudo service rsyslog restart 2024-09-20T22:05:27.747 INFO:teuthology.orchestra.run.smithi037.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-20T22:05:27.795 INFO:teuthology.orchestra.run.smithi194.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-20T22:05:28.301 INFO:teuthology.run_tasks:Running task internal.timer... 2024-09-20T22:05:28.314 INFO:teuthology.task.internal:Starting timer... 2024-09-20T22:05:28.315 INFO:teuthology.run_tasks:Running task pcp... 2024-09-20T22:05:28.340 INFO:teuthology.run_tasks:Running task selinux... 2024-09-20T22:05:28.363 DEBUG:teuthology.task:Applying overrides for task selinux: {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0', 'scontext=system_u:system_r:getty_t:s0']} 2024-09-20T22:05:28.363 DEBUG:teuthology.orchestra.run.smithi037:> sudo service auditd rotate 2024-09-20T22:05:28.423 INFO:teuthology.orchestra.run.smithi037.stdout:Rotating logs: 2024-09-20T22:05:28.424 DEBUG:teuthology.orchestra.run.smithi194:> sudo service auditd rotate 2024-09-20T22:05:28.490 INFO:teuthology.orchestra.run.smithi194.stdout:Rotating logs: 2024-09-20T22:05:28.492 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-09-20T22:05:28.492 DEBUG:teuthology.orchestra.run.smithi037:> /usr/sbin/getenforce 2024-09-20T22:05:28.526 INFO:teuthology.orchestra.run.smithi037.stdout:Permissive 2024-09-20T22:05:28.526 DEBUG:teuthology.orchestra.run.smithi194:> /usr/sbin/getenforce 2024-09-20T22:05:28.554 INFO:teuthology.orchestra.run.smithi194.stdout:Permissive 2024-09-20T22:05:28.554 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi037.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi194.front.sepia.ceph.com': 'permissive'} 2024-09-20T22:05:28.555 DEBUG:teuthology.orchestra.run.smithi037:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-09-20T22:05:28.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-20T22:05:28.593 DEBUG:teuthology.orchestra.run.smithi194:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-09-20T22:05:28.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-20T22:05:28.623 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-09-20T22:05:28.623 DEBUG:teuthology.orchestra.run.smithi037:> sudo /usr/sbin/setenforce permissive 2024-09-20T22:05:28.671 DEBUG:teuthology.orchestra.run.smithi194:> sudo /usr/sbin/setenforce permissive 2024-09-20T22:05:28.714 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-09-20T22:05:28.726 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2024-09-20T22:05:28.728 INFO:teuthology.repo_utils:Fetching git.ceph.com_ceph-cm-ansible_main from origin 2024-09-20T22:05:28.829 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-09-20T22:05:28.838 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-09-20T22:05:28.839 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi037.front.sepia.ceph.com,smithi194.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-09-20T22:09:15.313 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi037.front.sepia.ceph.com'), Remote(name='ubuntu@smithi194.front.sepia.ceph.com')] 2024-09-20T22:09:15.314 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi037.front.sepia.ceph.com' 2024-09-20T22:09:15.315 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi037.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-20T22:09:15.390 DEBUG:teuthology.orchestra.run.smithi037:> true 2024-09-20T22:09:15.474 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi037.front.sepia.ceph.com' 2024-09-20T22:09:15.474 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi194.front.sepia.ceph.com' 2024-09-20T22:09:15.475 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi194.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-20T22:09:15.549 DEBUG:teuthology.orchestra.run.smithi194:> true 2024-09-20T22:09:15.583 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi194.front.sepia.ceph.com' 2024-09-20T22:09:15.583 INFO:teuthology.run_tasks:Running task clock... 2024-09-20T22:09:15.597 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-09-20T22:09:15.597 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-20T22:09:15.597 DEBUG:teuthology.orchestra.run.smithi037:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-20T22:09:15.600 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-20T22:09:15.600 DEBUG:teuthology.orchestra.run.smithi194:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-20T22:09:15.627 INFO:teuthology.orchestra.run.smithi037.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-09-20T22:09:15.642 INFO:teuthology.orchestra.run.smithi037.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-09-20T22:09:15.651 INFO:teuthology.orchestra.run.smithi194.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-09-20T22:09:15.666 INFO:teuthology.orchestra.run.smithi194.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-09-20T22:09:15.668 INFO:teuthology.orchestra.run.smithi037.stderr:sudo: ntpd: command not found 2024-09-20T22:09:15.679 INFO:teuthology.orchestra.run.smithi037.stdout:506 Cannot talk to daemon 2024-09-20T22:09:15.689 INFO:teuthology.orchestra.run.smithi194.stderr:sudo: ntpd: command not found 2024-09-20T22:09:15.694 INFO:teuthology.orchestra.run.smithi037.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-09-20T22:09:15.700 INFO:teuthology.orchestra.run.smithi194.stdout:506 Cannot talk to daemon 2024-09-20T22:09:15.709 INFO:teuthology.orchestra.run.smithi037.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-09-20T22:09:15.715 INFO:teuthology.orchestra.run.smithi194.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-09-20T22:09:15.729 INFO:teuthology.orchestra.run.smithi194.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-09-20T22:09:15.772 INFO:teuthology.orchestra.run.smithi037.stderr:bash: line 1: ntpq: command not found 2024-09-20T22:09:15.776 INFO:teuthology.orchestra.run.smithi037.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-20T22:09:15.776 INFO:teuthology.orchestra.run.smithi037.stdout:=============================================================================== 2024-09-20T22:09:15.777 INFO:teuthology.orchestra.run.smithi037.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-20T22:09:15.777 INFO:teuthology.orchestra.run.smithi037.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-20T22:09:15.777 INFO:teuthology.orchestra.run.smithi037.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-20T22:09:15.777 INFO:teuthology.orchestra.run.smithi037.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-20T22:09:15.783 INFO:teuthology.orchestra.run.smithi194.stderr:bash: line 1: ntpq: command not found 2024-09-20T22:09:15.787 INFO:teuthology.orchestra.run.smithi194.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-20T22:09:15.787 INFO:teuthology.orchestra.run.smithi194.stdout:=============================================================================== 2024-09-20T22:09:15.787 INFO:teuthology.orchestra.run.smithi194.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-20T22:09:15.787 INFO:teuthology.orchestra.run.smithi194.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-20T22:09:15.787 INFO:teuthology.orchestra.run.smithi194.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-20T22:09:15.788 INFO:teuthology.orchestra.run.smithi194.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-20T22:09:15.788 INFO:teuthology.run_tasks:Running task cephadm... 2024-09-20T22:09:15.879 INFO:tasks.cephadm:Config: {'cephadm_branch': 'v17.2.0', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'image': 'quay.io/ceph/ceph:v17.2.0', 'roleless': True, 'conf': {'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', 'CEPHADM_REFRESH_FAILED'], 'log-only-match': ['CEPHADM_'], 'sha1': '9e2ac70bb7aea67d34f0d50002344abbe7803736'} 2024-09-20T22:09:15.879 INFO:tasks.cephadm:Cluster image is quay.io/ceph/ceph:v17.2.0 2024-09-20T22:09:15.879 INFO:tasks.cephadm:Cluster fsid is f9a6357a-779c-11ef-baf5-efdc52797490 2024-09-20T22:09:15.879 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-09-20T22:09:15.880 INFO:tasks.cephadm:No mon roles; fabricating mons 2024-09-20T22:09:15.880 INFO:tasks.cephadm:Monitor IPs: {'mon.smithi037': '172.21.15.37', 'mon.smithi194': '172.21.15.194'} 2024-09-20T22:09:15.880 INFO:tasks.cephadm:Normalizing hostnames... 2024-09-20T22:09:15.880 DEBUG:teuthology.orchestra.run.smithi037:> sudo hostname $(hostname -s) 2024-09-20T22:09:15.907 DEBUG:teuthology.orchestra.run.smithi194:> sudo hostname $(hostname -s) 2024-09-20T22:09:15.933 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref v17.2.0)... 2024-09-20T22:09:15.933 DEBUG:teuthology.orchestra.run.smithi037:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-09-20T22:09:16.410 INFO:teuthology.orchestra.run.smithi037.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Sep 20 22:09 /home/ubuntu/cephtest/cephadm 2024-09-20T22:09:16.411 DEBUG:teuthology.orchestra.run.smithi194:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-09-20T22:09:16.653 INFO:teuthology.orchestra.run.smithi194.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Sep 20 22:09 /home/ubuntu/cephtest/cephadm 2024-09-20T22:09:16.654 DEBUG:teuthology.orchestra.run.smithi037:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-09-20T22:09:16.672 DEBUG:teuthology.orchestra.run.smithi194:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-09-20T22:09:16.696 INFO:tasks.cephadm:Pulling image quay.io/ceph/ceph:v17.2.0 on all hosts... 2024-09-20T22:09:16.697 DEBUG:teuthology.orchestra.run.smithi037:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-09-20T22:09:16.714 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-09-20T22:09:18.286 INFO:teuthology.orchestra.run.smithi194.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-09-20T22:09:18.463 INFO:teuthology.orchestra.run.smithi037.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-09-20T22:09:30.048 INFO:teuthology.orchestra.run.smithi194.stdout:{ 2024-09-20T22:09:30.048 INFO:teuthology.orchestra.run.smithi194.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-09-20T22:09:30.048 INFO:teuthology.orchestra.run.smithi194.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-09-20T22:09:30.048 INFO:teuthology.orchestra.run.smithi194.stdout: "repo_digests": [ 2024-09-20T22:09:30.049 INFO:teuthology.orchestra.run.smithi194.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-09-20T22:09:30.049 INFO:teuthology.orchestra.run.smithi194.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-09-20T22:09:30.049 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2024-09-20T22:09:30.049 INFO:teuthology.orchestra.run.smithi194.stdout:} 2024-09-20T22:09:30.246 INFO:teuthology.orchestra.run.smithi037.stdout:{ 2024-09-20T22:09:30.246 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-09-20T22:09:30.246 INFO:teuthology.orchestra.run.smithi037.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-09-20T22:09:30.247 INFO:teuthology.orchestra.run.smithi037.stdout: "repo_digests": [ 2024-09-20T22:09:30.247 INFO:teuthology.orchestra.run.smithi037.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-09-20T22:09:30.247 INFO:teuthology.orchestra.run.smithi037.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-09-20T22:09:30.247 INFO:teuthology.orchestra.run.smithi037.stdout: ] 2024-09-20T22:09:30.247 INFO:teuthology.orchestra.run.smithi037.stdout:} 2024-09-20T22:09:30.265 DEBUG:teuthology.orchestra.run.smithi037:> sudo mkdir -p /etc/ceph 2024-09-20T22:09:30.317 DEBUG:teuthology.orchestra.run.smithi194:> sudo mkdir -p /etc/ceph 2024-09-20T22:09:30.345 DEBUG:teuthology.orchestra.run.smithi037:> sudo chmod 777 /etc/ceph 2024-09-20T22:09:30.371 DEBUG:teuthology.orchestra.run.smithi194:> sudo chmod 777 /etc/ceph 2024-09-20T22:09:30.410 INFO:tasks.cephadm:Writing seed config... 2024-09-20T22:09:30.411 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-09-20T22:09:30.411 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-09-20T22:09:30.411 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-09-20T22:09:30.411 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-09-20T22:09:30.411 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-09-20T22:09:30.412 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-09-20T22:09:30.412 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-09-20T22:09:30.412 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2024-09-20T22:09:30.412 DEBUG:teuthology.orchestra.run.smithi037:> set -ex 2024-09-20T22:09:30.412 DEBUG:teuthology.orchestra.run.smithi037:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-09-20T22:09:30.428 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = f9a6357a-779c-11ef-baf5-efdc52797490 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-09-20T22:09:30.428 DEBUG:teuthology.orchestra.run.smithi037:mon.smithi037> sudo journalctl -f -n 0 -u ceph-f9a6357a-779c-11ef-baf5-efdc52797490@mon.smithi037.service 2024-09-20T22:09:30.470 INFO:tasks.cephadm:Bootstrapping... 2024-09-20T22:09:30.471 DEBUG:teuthology.orchestra.run.smithi037:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 -v bootstrap --fsid f9a6357a-779c-11ef-baf5-efdc52797490 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-ip 172.21.15.37 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-09-20T22:09:30.667 INFO:teuthology.orchestra.run.smithi037.stderr:-------------------------------------------------------------------------------- 2024-09-20T22:09:30.667 INFO:teuthology.orchestra.run.smithi037.stderr:cephadm ['--image', 'quay.io/ceph/ceph:v17.2.0', '-v', 'bootstrap', '--fsid', 'f9a6357a-779c-11ef-baf5-efdc52797490', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-ip', '172.21.15.37', '--skip-admin-label'] 2024-09-20T22:09:30.689 INFO:teuthology.orchestra.run.smithi037.stderr:/bin/podman: 5.2.2 2024-09-20T22:09:30.691 INFO:teuthology.orchestra.run.smithi037.stderr:Verifying podman|docker is present... 2024-09-20T22:09:30.712 INFO:teuthology.orchestra.run.smithi037.stderr:/bin/podman: 5.2.2 2024-09-20T22:09:30.714 INFO:teuthology.orchestra.run.smithi037.stderr:Verifying lvm2 is present... 2024-09-20T22:09:30.714 INFO:teuthology.orchestra.run.smithi037.stderr:Verifying time synchronization is in place... 2024-09-20T22:09:30.721 INFO:teuthology.orchestra.run.smithi037.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-09-20T22:09:30.727 INFO:teuthology.orchestra.run.smithi037.stderr:systemctl: inactive 2024-09-20T22:09:30.734 INFO:teuthology.orchestra.run.smithi037.stderr:systemctl: enabled 2024-09-20T22:09:30.741 INFO:teuthology.orchestra.run.smithi037.stderr:systemctl: active 2024-09-20T22:09:30.741 INFO:teuthology.orchestra.run.smithi037.stderr:Unit chronyd.service is enabled and running 2024-09-20T22:09:30.741 INFO:teuthology.orchestra.run.smithi037.stderr:Repeating the final host check... 2024-09-20T22:09:30.763 INFO:teuthology.orchestra.run.smithi037.stderr:/bin/podman: 5.2.2 2024-09-20T22:09:30.764 INFO:teuthology.orchestra.run.smithi037.stderr:podman (/bin/podman) version 5.2.2 is present 2024-09-20T22:09:30.764 INFO:teuthology.orchestra.run.smithi037.stderr:systemctl is present 2024-09-20T22:09:30.764 INFO:teuthology.orchestra.run.smithi037.stderr:lvcreate is present 2024-09-20T22:09:30.770 INFO:teuthology.orchestra.run.smithi037.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-09-20T22:09:30.777 INFO:teuthology.orchestra.run.smithi037.stderr:systemctl: inactive 2024-09-20T22:09:30.784 INFO:teuthology.orchestra.run.smithi037.stderr:systemctl: enabled 2024-09-20T22:09:30.790 INFO:teuthology.orchestra.run.smithi037.stderr:systemctl: active 2024-09-20T22:09:30.791 INFO:teuthology.orchestra.run.smithi037.stderr:Unit chronyd.service is enabled and running 2024-09-20T22:09:30.791 INFO:teuthology.orchestra.run.smithi037.stderr:Host looks OK 2024-09-20T22:09:30.791 INFO:teuthology.orchestra.run.smithi037.stderr:Cluster fsid: f9a6357a-779c-11ef-baf5-efdc52797490 2024-09-20T22:09:30.791 INFO:teuthology.orchestra.run.smithi037.stderr:Acquiring lock 140661943879568 on /run/cephadm/f9a6357a-779c-11ef-baf5-efdc52797490.lock 2024-09-20T22:09:30.792 INFO:teuthology.orchestra.run.smithi037.stderr:Lock 140661943879568 acquired on /run/cephadm/f9a6357a-779c-11ef-baf5-efdc52797490.lock 2024-09-20T22:09:30.792 INFO:teuthology.orchestra.run.smithi037.stderr:Verifying IP 172.21.15.37 port 3300 ... 2024-09-20T22:09:30.792 INFO:teuthology.orchestra.run.smithi037.stderr:Verifying IP 172.21.15.37 port 6789 ... 2024-09-20T22:09:30.792 INFO:teuthology.orchestra.run.smithi037.stderr:Base mon IP is 172.21.15.37, final addrv is [v2:172.21.15.37:3300,v1:172.21.15.37:6789] 2024-09-20T22:09:30.795 INFO:teuthology.orchestra.run.smithi037.stderr:/sbin/ip: default via 172.21.15.254 dev ens1f0 proto dhcp src 172.21.15.37 metric 100 2024-09-20T22:09:30.795 INFO:teuthology.orchestra.run.smithi037.stderr:/sbin/ip: 172.21.0.0/20 dev ens1f0 proto kernel scope link src 172.21.15.37 metric 100 2024-09-20T22:09:30.797 INFO:teuthology.orchestra.run.smithi037.stderr:/sbin/ip: ::1 dev lo proto kernel metric 256 pref medium 2024-09-20T22:09:30.798 INFO:teuthology.orchestra.run.smithi037.stderr:/sbin/ip: fe80::/64 dev ens1f0 proto kernel metric 256 pref medium 2024-09-20T22:09:30.798 INFO:teuthology.orchestra.run.smithi037.stderr:/sbin/ip: default via fe80::327c:5e00:6487:71e0 dev ens1f0 proto ra metric 1024 expires 1774sec hoplimit 64 pref medium 2024-09-20T22:09:30.800 INFO:teuthology.orchestra.run.smithi037.stderr:/sbin/ip: 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-09-20T22:09:30.800 INFO:teuthology.orchestra.run.smithi037.stderr:/sbin/ip: inet6 ::1/128 scope host 2024-09-20T22:09:30.800 INFO:teuthology.orchestra.run.smithi037.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-09-20T22:09:30.800 INFO:teuthology.orchestra.run.smithi037.stderr:/sbin/ip: 4: ens1f0: mtu 1500 state UP qlen 1000 2024-09-20T22:09:30.801 INFO:teuthology.orchestra.run.smithi037.stderr:/sbin/ip: inet6 fe80::ec4:7aff:fe88:7794/64 scope link 2024-09-20T22:09:30.801 INFO:teuthology.orchestra.run.smithi037.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-09-20T22:09:30.801 INFO:teuthology.orchestra.run.smithi037.stderr:Mon IP `172.21.15.37` is in CIDR network `172.21.0.0/20` 2024-09-20T22:09:30.801 INFO:teuthology.orchestra.run.smithi037.stderr:- internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-09-20T22:09:30.802 INFO:teuthology.orchestra.run.smithi037.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-09-20T22:09:30.904 INFO:teuthology.orchestra.run.smithi037.stderr:/bin/podman: Trying to pull quay.io/ceph/ceph:v17.2.0... 2024-09-20T22:09:31.285 INFO:teuthology.orchestra.run.smithi037.stderr:/bin/podman: Getting image source signatures 2024-09-20T22:09:31.285 INFO:teuthology.orchestra.run.smithi037.stderr:/bin/podman: Copying blob sha256:89b4a75bc2d8500f15463747507c9623df43886c134463e7f0527e70900e7a7b 2024-09-20T22:09:31.285 INFO:teuthology.orchestra.run.smithi037.stderr:/bin/podman: Copying blob sha256:33ca8fff7868c4dc0c11e09bca97c720eb9cfbab7221216754367dd8de70388a 2024-09-20T22:09:31.285 INFO:teuthology.orchestra.run.smithi037.stderr:/bin/podman: Copying blob sha256:c32ab78b488d0c72f64eded765c0cf6b5bf2c75dab66cb62a9d367fa6ec42513 2024-09-20T22:09:31.285 INFO:teuthology.orchestra.run.smithi037.stderr:/bin/podman: Copying blob sha256:599d07cb321ff0a3c82224e1138fc685793fa69b93ed5780415751a5f7e4b8c2 2024-09-20T22:09:31.285 INFO:teuthology.orchestra.run.smithi037.stderr:/bin/podman: Copying blob sha256:a70843738bb77e1ab9c1f85969ebdfa55f178e746be081d1cb4f94011f69eb7c 2024-09-20T22:09:31.286 INFO:teuthology.orchestra.run.smithi037.stderr:/bin/podman: Copying config sha256:e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-09-20T22:09:31.450 INFO:teuthology.orchestra.run.smithi037.stderr:/bin/podman: Writing manifest to image destination 2024-09-20T22:09:31.605 INFO:teuthology.orchestra.run.smithi037.stderr:/bin/podman: e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-09-20T22:09:32.025 INFO:teuthology.orchestra.run.smithi037.stderr:ceph: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-09-20T22:09:32.577 INFO:teuthology.orchestra.run.smithi037.stderr:Ceph version: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-09-20T22:09:32.577 INFO:teuthology.orchestra.run.smithi037.stderr:Extracting ceph user uid/gid from container image... 2024-09-20T22:09:32.993 INFO:teuthology.orchestra.run.smithi037.stderr:stat: 167 167 2024-09-20T22:09:33.508 INFO:teuthology.orchestra.run.smithi037.stderr:Creating initial keys... 2024-09-20T22:09:33.939 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph-authtool: AQCd8u1m3N79MhAASkOLMwwLa8uEEdBWX3EMig== 2024-09-20T22:09:35.011 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph-authtool: AQCe8u1muVpHNxAAg6nGf/1cybJsGrMiFJ2iew== 2024-09-20T22:09:36.059 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph-authtool: AQCf8u1mTeqeOhAACd4ei+vg0sG7YoJ2evIaaQ== 2024-09-20T22:09:36.558 INFO:teuthology.orchestra.run.smithi037.stderr:Creating initial monmap... 2024-09-20T22:09:36.956 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: monmap file /tmp/monmap 2024-09-20T22:09:36.956 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/monmaptool: setting min_mon_release = octopus 2024-09-20T22:09:36.956 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: set fsid to f9a6357a-779c-11ef-baf5-efdc52797490 2024-09-20T22:09:36.956 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-09-20T22:09:37.479 INFO:teuthology.orchestra.run.smithi037.stderr:monmaptool for smithi037 [v2:172.21.15.37:3300,v1:172.21.15.37:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-09-20T22:09:37.480 INFO:teuthology.orchestra.run.smithi037.stderr:setting min_mon_release = octopus 2024-09-20T22:09:37.480 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/monmaptool: set fsid to f9a6357a-779c-11ef-baf5-efdc52797490 2024-09-20T22:09:37.480 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-09-20T22:09:37.480 INFO:teuthology.orchestra.run.smithi037.stderr: 2024-09-20T22:09:37.480 INFO:teuthology.orchestra.run.smithi037.stderr:Creating mon... 2024-09-20T22:09:38.435 INFO:teuthology.orchestra.run.smithi037.stderr:create mon.smithi037 on 2024-09-20T22:09:38.717 INFO:teuthology.orchestra.run.smithi037.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-09-20T22:09:38.988 INFO:teuthology.orchestra.run.smithi037.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-f9a6357a-779c-11ef-baf5-efdc52797490.target → /etc/systemd/system/ceph-f9a6357a-779c-11ef-baf5-efdc52797490.target. 2024-09-20T22:09:38.988 INFO:teuthology.orchestra.run.smithi037.stderr:systemctl: Created symlink /etc/systemd/system/ceph.target.wants/ceph-f9a6357a-779c-11ef-baf5-efdc52797490.target → /etc/systemd/system/ceph-f9a6357a-779c-11ef-baf5-efdc52797490.target. 2024-09-20T22:09:39.746 INFO:teuthology.orchestra.run.smithi037.stderr:systemctl: Failed to reset failed state of unit ceph-f9a6357a-779c-11ef-baf5-efdc52797490@mon.smithi037.service: Unit ceph-f9a6357a-779c-11ef-baf5-efdc52797490@mon.smithi037.service not loaded. 2024-09-20T22:09:39.754 INFO:teuthology.orchestra.run.smithi037.stderr:systemctl: Created symlink /etc/systemd/system/ceph-f9a6357a-779c-11ef-baf5-efdc52797490.target.wants/ceph-f9a6357a-779c-11ef-baf5-efdc52797490@mon.smithi037.service → /etc/systemd/system/ceph-f9a6357a-779c-11ef-baf5-efdc52797490@.service. 2024-09-20T22:09:40.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:40 smithi037 podman[23357]: 2024-09-20 22:09:40.188807912 +0000 UTC m=+0.017068764 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-09-20T22:09:41.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:41 smithi037 podman[23357]: 2024-09-20 22:09:41.289652091 +0000 UTC m=+1.117912933 container create e7488572675f981b8f46804eeef5fc98c72cf26151bc8cb3ec915906cb56cd21 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-mon-smithi037, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , GIT_CLEAN=True, release=754, build-date=2022-05-03T08:36:31.336870, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, name=centos-stream, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, distribution-scope=public, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-09-20T22:09:43.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:43 smithi037 podman[23357]: 2024-09-20 22:09:43.072479321 +0000 UTC m=+2.900740172 container init e7488572675f981b8f46804eeef5fc98c72cf26151bc8cb3ec915906cb56cd21 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-mon-smithi037, io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, ceph=True, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, release=754, maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, distribution-scope=public, version=8, vendor=Red Hat, Inc., io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container) 2024-09-20T22:09:43.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:43 smithi037 podman[23357]: 2024-09-20 22:09:43.076551455 +0000 UTC m=+2.904812298 container start e7488572675f981b8f46804eeef5fc98c72cf26151bc8cb3ec915906cb56cd21 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-mon-smithi037, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, name=centos-stream, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public) 2024-09-20T22:09:43.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:43 smithi037 ceph-mon[23375]: mkfs f9a6357a-779c-11ef-baf5-efdc52797490 2024-09-20T22:09:43.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:43 smithi037 ceph-mon[23375]: mon.smithi037 is new leader, mons smithi037 in quorum (ranks 0) 2024-09-20T22:09:44.009 INFO:teuthology.orchestra.run.smithi037.stderr:systemctl: disabled 2024-09-20T22:09:44.018 INFO:teuthology.orchestra.run.smithi037.stderr:systemctl: inactive 2024-09-20T22:09:44.018 INFO:teuthology.orchestra.run.smithi037.stderr:firewalld.service is not enabled 2024-09-20T22:09:44.018 INFO:teuthology.orchestra.run.smithi037.stderr:Not possible to enable service . firewalld.service is not available 2024-09-20T22:09:44.019 INFO:teuthology.orchestra.run.smithi037.stderr:Waiting for mon to start... 2024-09-20T22:09:44.019 INFO:teuthology.orchestra.run.smithi037.stderr:Waiting for mon... 2024-09-20T22:09:44.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:43 smithi037 bash[23357]: e7488572675f981b8f46804eeef5fc98c72cf26151bc8cb3ec915906cb56cd21 2024-09-20T22:09:44.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:43 smithi037 systemd[1]: Started Ceph mon.smithi037 for f9a6357a-779c-11ef-baf5-efdc52797490. 2024-09-20T22:09:44.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:44 smithi037 ceph-mon[23375]: mon.smithi037 is new leader, mons smithi037 in quorum (ranks 0) 2024-09-20T22:09:44.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:44 smithi037 ceph-mon[23375]: monmap e1: 1 mons at {smithi037=[v2:172.21.15.37:3300/0,v1:172.21.15.37:6789/0]} 2024-09-20T22:09:44.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:44 smithi037 ceph-mon[23375]: fsmap 2024-09-20T22:09:44.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:44 smithi037 ceph-mon[23375]: osdmap e1: 0 total, 0 up, 0 in 2024-09-20T22:09:44.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:44 smithi037 ceph-mon[23375]: mgrmap e1: no daemons active 2024-09-20T22:09:47.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:47 smithi037 ceph-mon[23375]: from='client.? 172.21.15.37:0/1043108098' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-20T22:09:47.895 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: cluster: 2024-09-20T22:09:47.896 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: id: f9a6357a-779c-11ef-baf5-efdc52797490 2024-09-20T22:09:47.896 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: health: HEALTH_OK 2024-09-20T22:09:47.896 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: 2024-09-20T22:09:47.896 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: services: 2024-09-20T22:09:47.896 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: mon: 1 daemons, quorum smithi037 (age 3s) 2024-09-20T22:09:47.896 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: mgr: no daemons active 2024-09-20T22:09:47.896 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: osd: 0 osds: 0 up, 0 in 2024-09-20T22:09:47.896 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: 2024-09-20T22:09:47.896 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: data: 2024-09-20T22:09:47.896 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: pools: 0 pools, 0 pgs 2024-09-20T22:09:47.896 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: objects: 0 objects, 0 B 2024-09-20T22:09:47.896 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: usage: 0 B used, 0 B / 0 B avail 2024-09-20T22:09:47.896 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: pgs: 2024-09-20T22:09:47.896 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: 2024-09-20T22:09:50.130 INFO:teuthology.orchestra.run.smithi037.stderr:mon is available 2024-09-20T22:09:50.130 INFO:teuthology.orchestra.run.smithi037.stderr:Assimilating anything we can from ceph.conf... 2024-09-20T22:09:50.651 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: 2024-09-20T22:09:50.651 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: [global] 2024-09-20T22:09:50.651 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: fsid = f9a6357a-779c-11ef-baf5-efdc52797490 2024-09-20T22:09:50.651 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: mon_host = [v2:172.21.15.37:3300,v1:172.21.15.37:6789] 2024-09-20T22:09:50.651 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-09-20T22:09:50.651 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-09-20T22:09:50.651 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-09-20T22:09:50.651 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-09-20T22:09:50.651 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: 2024-09-20T22:09:50.652 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: [mgr] 2024-09-20T22:09:50.652 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-09-20T22:09:50.652 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: 2024-09-20T22:09:50.652 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: [osd] 2024-09-20T22:09:50.652 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-09-20T22:09:50.652 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-09-20T22:09:50.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:50 smithi037 ceph-mon[23375]: from='client.? 172.21.15.37:0/196369482' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-09-20T22:09:50.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:50 smithi037 ceph-mon[23375]: from='client.? 172.21.15.37:0/196369482' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-09-20T22:09:51.244 INFO:teuthology.orchestra.run.smithi037.stderr:Generating new minimal ceph.conf... 2024-09-20T22:09:51.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:51 smithi037 ceph-mon[23375]: from='client.? 172.21.15.37:0/329856369' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:09:52.317 INFO:teuthology.orchestra.run.smithi037.stderr:Restarting the monitor... 2024-09-20T22:09:52.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:52 smithi037 systemd[1]: Stopping Ceph mon.smithi037 for f9a6357a-779c-11ef-baf5-efdc52797490... 2024-09-20T22:09:52.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:52 smithi037 bash[23596]: Error: no container with name or ID "ceph-f9a6357a-779c-11ef-baf5-efdc52797490-mon.smithi037" found: no such container 2024-09-20T22:09:52.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:52 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-mon-smithi037[23371]: 2024-09-20T22:09:52.483+0000 7f31266a4700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi037 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-20T22:09:52.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:52 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-mon-smithi037[23371]: 2024-09-20T22:09:52.483+0000 7f31266a4700 -1 mon.smithi037@0(leader) e1 *** Got Signal Terminated *** 2024-09-20T22:09:52.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:52 smithi037 podman[23608]: 2024-09-20 22:09:52.53503848 +0000 UTC m=+0.161480915 container died e7488572675f981b8f46804eeef5fc98c72cf26151bc8cb3ec915906cb56cd21 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-mon-smithi037, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, vcs-type=git, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux ) 2024-09-20T22:09:53.068 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:52 smithi037 podman[23608]: 2024-09-20 22:09:52.874653719 +0000 UTC m=+0.501096157 container cleanup e7488572675f981b8f46804eeef5fc98c72cf26151bc8cb3ec915906cb56cd21 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-mon-smithi037, io.openshift.expose-services=, release=754, GIT_BRANCH=HEAD, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, ceph=True, name=centos-stream, vcs-type=git, GIT_CLEAN=True, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, version=8, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-09-20T22:09:53.068 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:52 smithi037 bash[23608]: ceph-f9a6357a-779c-11ef-baf5-efdc52797490-mon-smithi037 2024-09-20T22:09:53.327 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 podman[23621]: 2024-09-20 22:09:53.067086272 +0000 UTC m=+0.524461982 container remove e7488572675f981b8f46804eeef5fc98c72cf26151bc8cb3ec915906cb56cd21 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-mon-smithi037, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, distribution-scope=public, vendor=Red Hat, Inc., io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux ) 2024-09-20T22:09:53.327 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 bash[23635]: Error: no container with name or ID "ceph-f9a6357a-779c-11ef-baf5-efdc52797490-mon.smithi037" found: no such container 2024-09-20T22:09:53.327 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 systemd[1]: ceph-f9a6357a-779c-11ef-baf5-efdc52797490@mon.smithi037.service: Deactivated successfully. 2024-09-20T22:09:53.327 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 systemd[1]: Stopped Ceph mon.smithi037 for f9a6357a-779c-11ef-baf5-efdc52797490. 2024-09-20T22:09:53.327 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 systemd[1]: Starting Ceph mon.smithi037 for f9a6357a-779c-11ef-baf5-efdc52797490... 2024-09-20T22:09:53.587 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 podman[23699]: 2024-09-20 22:09:53.229794268 +0000 UTC m=+0.017315985 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-09-20T22:09:53.587 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 podman[23699]: 2024-09-20 22:09:53.34819315 +0000 UTC m=+0.135714854 container create 71f591122786ae60b80a25133890317cf9cad97c3c7ba98976cff6c0bb977f18 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-mon-smithi037, io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, RELEASE=HEAD, name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, ceph=True, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container) 2024-09-20T22:09:53.587 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 podman[23699]: 2024-09-20 22:09:53.562046001 +0000 UTC m=+0.349567718 container init 71f591122786ae60b80a25133890317cf9cad97c3c7ba98976cff6c0bb977f18 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-mon-smithi037, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, RELEASE=HEAD, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, vendor=Red Hat, Inc., GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, architecture=x86_64, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=) 2024-09-20T22:09:53.587 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 podman[23699]: 2024-09-20 22:09:53.565334004 +0000 UTC m=+0.352855709 container start 71f591122786ae60b80a25133890317cf9cad97c3c7ba98976cff6c0bb977f18 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-mon-smithi037, io.openshift.tags=base centos centos-stream, ceph=True, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, io.k8s.display-name=CentOS Stream 8, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, io.openshift.expose-services=, distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream) 2024-09-20T22:09:53.587 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: set uid:gid to 167:167 (ceph:ceph) 2024-09-20T22:09:53.587 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-09-20T22:09:53.587 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: pidfile_write: ignore empty --pid-file 2024-09-20T22:09:53.641 INFO:teuthology.orchestra.run.smithi037.stderr:Setting mon public_network to 172.21.0.0/20 2024-09-20T22:09:53.890 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: load: jerasure load: lrc 2024-09-20T22:09:53.890 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: RocksDB version: 6.15.5 2024-09-20T22:09:53.890 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-09-20T22:09:53.890 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Compile date Apr 18 2022 2024-09-20T22:09:53.890 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: DB SUMMARY 2024-09-20T22:09:53.891 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: DB Session ID: XJE6XVINWZS7FFWG0IME 2024-09-20T22:09:53.891 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: CURRENT file: CURRENT 2024-09-20T22:09:53.891 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: IDENTITY file: IDENTITY 2024-09-20T22:09:53.891 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-09-20T22:09:53.891 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi037/store.db dir, Total Num: 1, files: 000008.sst 2024-09-20T22:09:53.891 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi037/store.db: 000010.log size: 83499 ; 2024-09-20T22:09:53.891 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.error_if_exists: 0 2024-09-20T22:09:53.891 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.create_if_missing: 0 2024-09-20T22:09:53.891 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.paranoid_checks: 1 2024-09-20T22:09:53.891 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-20T22:09:53.891 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.env: 0x5568632c6860 2024-09-20T22:09:53.891 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.fs: Posix File System 2024-09-20T22:09:53.891 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.info_log: 0x5568640ebfa0 2024-09-20T22:09:53.891 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.max_file_opening_threads: 16 2024-09-20T22:09:53.891 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.statistics: (nil) 2024-09-20T22:09:53.891 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.use_fsync: 0 2024-09-20T22:09:53.892 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.max_log_file_size: 0 2024-09-20T22:09:53.892 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-20T22:09:53.892 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-20T22:09:53.892 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.keep_log_file_num: 1000 2024-09-20T22:09:53.892 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.recycle_log_file_num: 0 2024-09-20T22:09:53.892 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.allow_fallocate: 1 2024-09-20T22:09:53.892 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.allow_mmap_reads: 0 2024-09-20T22:09:53.892 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.allow_mmap_writes: 0 2024-09-20T22:09:53.892 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.use_direct_reads: 0 2024-09-20T22:09:53.892 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-20T22:09:53.892 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.create_missing_column_families: 0 2024-09-20T22:09:53.892 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.db_log_dir: 2024-09-20T22:09:53.892 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-smithi037/store.db 2024-09-20T22:09:53.892 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-20T22:09:53.892 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-20T22:09:53.892 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-20T22:09:53.893 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-20T22:09:53.893 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-20T22:09:53.894 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-20T22:09:53.894 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.advise_random_on_open: 1 2024-09-20T22:09:53.894 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.db_write_buffer_size: 0 2024-09-20T22:09:53.894 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.write_buffer_manager: 0x5568641dc330 2024-09-20T22:09:53.894 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-20T22:09:53.894 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-09-20T22:09:53.894 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-20T22:09:53.894 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-20T22:09:53.894 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.rate_limiter: (nil) 2024-09-20T22:09:53.894 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-20T22:09:53.895 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.wal_recovery_mode: 2 2024-09-20T22:09:53.895 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.enable_thread_tracking: 0 2024-09-20T22:09:53.895 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.enable_pipelined_write: 0 2024-09-20T22:09:53.895 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.unordered_write: 0 2024-09-20T22:09:53.895 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-20T22:09:53.895 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-20T22:09:53.895 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-20T22:09:53.895 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-20T22:09:53.895 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.row_cache: None 2024-09-20T22:09:53.895 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.wal_filter: None 2024-09-20T22:09:53.895 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-20T22:09:53.895 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.allow_ingest_behind: 0 2024-09-20T22:09:53.895 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.preserve_deletes: 0 2024-09-20T22:09:53.895 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.two_write_queues: 0 2024-09-20T22:09:53.895 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.manual_wal_flush: 0 2024-09-20T22:09:53.895 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.atomic_flush: 0 2024-09-20T22:09:53.896 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-20T22:09:53.896 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-20T22:09:53.896 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-20T22:09:53.896 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.log_readahead_size: 0 2024-09-20T22:09:53.896 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-20T22:09:53.896 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.best_efforts_recovery: 0 2024-09-20T22:09:53.896 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-20T22:09:53.896 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-20T22:09:53.896 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.allow_data_in_errors: 0 2024-09-20T22:09:53.896 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.db_host_id: __hostname__ 2024-09-20T22:09:53.896 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.max_background_jobs: 2 2024-09-20T22:09:53.896 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.max_background_compactions: -1 2024-09-20T22:09:53.896 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.max_subcompactions: 1 2024-09-20T22:09:53.896 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-20T22:09:53.896 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-20T22:09:53.897 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-20T22:09:53.897 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.max_total_wal_size: 0 2024-09-20T22:09:53.897 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-20T22:09:53.897 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-20T22:09:53.897 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-20T22:09:53.897 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-20T22:09:53.897 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.max_open_files: -1 2024-09-20T22:09:53.897 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.bytes_per_sync: 0 2024-09-20T22:09:53.897 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-20T22:09:53.897 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-20T22:09:53.897 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.compaction_readahead_size: 0 2024-09-20T22:09:53.897 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.max_background_flushes: -1 2024-09-20T22:09:53.897 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Compression algorithms supported: 2024-09-20T22:09:53.897 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-20T22:09:53.897 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: kZSTD supported: 0 2024-09-20T22:09:53.897 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: kXpressCompression supported: 0 2024-09-20T22:09:53.898 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: kLZ4HCCompression supported: 1 2024-09-20T22:09:53.898 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: kLZ4Compression supported: 1 2024-09-20T22:09:53.898 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: kBZip2Compression supported: 0 2024-09-20T22:09:53.898 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: kZlibCompression supported: 1 2024-09-20T22:09:53.898 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: kSnappyCompression supported: 1 2024-09-20T22:09:53.898 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-20T22:09:53.898 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi037/store.db/MANIFEST-000009 2024-09-20T22:09:53.898 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-09-20T22:09:53.898 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-20T22:09:53.898 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.merge_operator: 2024-09-20T22:09:53.898 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.compaction_filter: None 2024-09-20T22:09:53.898 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.compaction_filter_factory: None 2024-09-20T22:09:53.898 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.sst_partitioner_factory: None 2024-09-20T22:09:53.898 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-20T22:09:53.899 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-20T22:09:53.899 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5568640b7d00) 2024-09-20T22:09:53.899 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: cache_index_and_filter_blocks: 1 2024-09-20T22:09:53.899 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-20T22:09:53.899 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-20T22:09:53.899 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: pin_top_level_index_and_filter: 1 2024-09-20T22:09:53.899 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: index_type: 0 2024-09-20T22:09:53.899 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: data_block_index_type: 0 2024-09-20T22:09:53.899 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: index_shortening: 1 2024-09-20T22:09:53.899 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-20T22:09:53.899 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: hash_index_allow_collision: 1 2024-09-20T22:09:53.899 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: checksum: 1 2024-09-20T22:09:53.899 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: no_block_cache: 0 2024-09-20T22:09:53.899 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: block_cache: 0x556864122170 2024-09-20T22:09:53.900 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: block_cache_name: BinnedLRUCache 2024-09-20T22:09:53.900 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: block_cache_options: 2024-09-20T22:09:53.900 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: capacity : 536870912 2024-09-20T22:09:53.900 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: num_shard_bits : 4 2024-09-20T22:09:53.900 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: strict_capacity_limit : 0 2024-09-20T22:09:53.900 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: high_pri_pool_ratio: 0.000 2024-09-20T22:09:53.900 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: block_cache_compressed: (nil) 2024-09-20T22:09:53.900 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: persistent_cache: (nil) 2024-09-20T22:09:53.900 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: block_size: 4096 2024-09-20T22:09:53.900 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: block_size_deviation: 10 2024-09-20T22:09:53.900 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: block_restart_interval: 16 2024-09-20T22:09:53.900 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: index_block_restart_interval: 1 2024-09-20T22:09:53.900 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: metadata_block_size: 4096 2024-09-20T22:09:53.900 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: partition_filters: 0 2024-09-20T22:09:53.900 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: use_delta_encoding: 1 2024-09-20T22:09:53.901 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-09-20T22:09:53.901 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: whole_key_filtering: 1 2024-09-20T22:09:53.901 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: verify_compression: 0 2024-09-20T22:09:53.901 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: read_amp_bytes_per_bit: 0 2024-09-20T22:09:53.901 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: format_version: 4 2024-09-20T22:09:53.901 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: enable_index_compression: 1 2024-09-20T22:09:53.901 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: block_align: 0 2024-09-20T22:09:53.901 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.write_buffer_size: 33554432 2024-09-20T22:09:53.901 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.max_write_buffer_number: 2 2024-09-20T22:09:53.901 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.compression: NoCompression 2024-09-20T22:09:53.901 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.bottommost_compression: Disabled 2024-09-20T22:09:53.901 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.prefix_extractor: nullptr 2024-09-20T22:09:53.901 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-20T22:09:53.901 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.num_levels: 7 2024-09-20T22:09:53.901 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-20T22:09:53.902 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-20T22:09:53.902 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-20T22:09:53.902 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-20T22:09:53.902 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-20T22:09:53.902 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-20T22:09:53.902 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-20T22:09:53.902 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-20T22:09:53.902 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-20T22:09:53.902 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-20T22:09:53.902 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-20T22:09:53.902 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.compression_opts.level: 32767 2024-09-20T22:09:53.902 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.compression_opts.strategy: 0 2024-09-20T22:09:53.902 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-20T22:09:53.902 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-20T22:09:53.902 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-20T22:09:53.903 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.compression_opts.enabled: false 2024-09-20T22:09:53.903 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-20T22:09:53.903 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-20T22:09:53.903 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-20T22:09:53.903 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.target_file_size_base: 67108864 2024-09-20T22:09:53.903 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-20T22:09:53.903 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-20T22:09:53.903 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-20T22:09:53.903 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-20T22:09:53.903 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-20T22:09:53.903 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-20T22:09:53.903 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-20T22:09:53.903 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-20T22:09:53.904 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-20T22:09:53.904 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-20T22:09:53.904 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-20T22:09:53.904 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-20T22:09:53.904 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-20T22:09:53.904 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.arena_block_size: 4194304 2024-09-20T22:09:53.904 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-20T22:09:53.904 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-20T22:09:53.904 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-09-20T22:09:53.904 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.disable_auto_compactions: 0 2024-09-20T22:09:53.904 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-20T22:09:53.904 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-20T22:09:53.904 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-20T22:09:53.904 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-20T22:09:53.905 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-20T22:09:53.905 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-20T22:09:53.905 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-20T22:09:53.905 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-20T22:09:53.905 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-20T22:09:53.905 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-20T22:09:53.905 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.table_properties_collectors: 2024-09-20T22:09:53.905 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.inplace_update_support: 0 2024-09-20T22:09:53.905 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-20T22:09:53.905 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-20T22:09:53.905 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-20T22:09:53.905 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-20T22:09:53.905 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.bloom_locality: 0 2024-09-20T22:09:53.905 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.max_successive_merges: 0 2024-09-20T22:09:53.906 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-20T22:09:53.906 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.paranoid_file_checks: 0 2024-09-20T22:09:53.906 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.force_consistency_checks: 1 2024-09-20T22:09:53.906 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.report_bg_io_stats: 0 2024-09-20T22:09:53.906 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.ttl: 2592000 2024-09-20T22:09:53.906 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-20T22:09:53.906 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.enable_blob_files: false 2024-09-20T22:09:53.906 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.min_blob_size: 0 2024-09-20T22:09:53.906 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.blob_file_size: 268435456 2024-09-20T22:09:53.906 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-20T22:09:53.906 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-20T22:09:53.906 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-20T22:09:53.906 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi037/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-20T22:09:53.906 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 5 2024-09-20T22:09:53.907 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: [db/version_set.cc:4083] Creating manifest 13 2024-09-20T22:09:53.907 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726870193591936, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-09-20T22:09:53.907 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #10 mode 2 2024-09-20T22:09:53.907 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-09-20T22:09:53.907 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726870193592687, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 79827, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 78072, "index_size": 243, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 9930, "raw_average_key_size": 46, "raw_value_size": 72323, "raw_average_value_size": 341, "num_data_blocks": 11, "num_entries": 212, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1726870193, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "c3fa949c-238a-4d0b-8729-046e6c8205d9", "db_session_id": "XJE6XVINWZS7FFWG0IME"}} 2024-09-20T22:09:53.907 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: [db/version_set.cc:4083] Creating manifest 15 2024-09-20T22:09:53.907 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726870193593027, "job": 1, "event": "recovery_finished"} 2024-09-20T22:09:53.907 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-smithi037/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-20T22:09:53.907 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x556864108a80 2024-09-20T22:09:53.907 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: DB pointer 0x556864118000 2024-09-20T22:09:53.907 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-09-20T22:09:53.907 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-09-20T22:09:53.907 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: ** DB Stats ** 2024-09-20T22:09:53.907 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-20T22:09:53.908 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-20T22:09:53.908 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-20T22:09:53.908 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-20T22:09:53.908 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-20T22:09:53.908 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-09-20T22:09:53.908 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-20T22:09:53.908 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: 2024-09-20T22:09:53.908 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: ** Compaction Stats [default] ** 2024-09-20T22:09:53.908 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-20T22:09:53.908 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-20T22:09:53.908 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: L0 2/0 79.75 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 158.9 0.00 0.00 1 0.000 0 0 2024-09-20T22:09:53.908 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: Sum 2/0 79.75 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 158.9 0.00 0.00 1 0.000 0 0 2024-09-20T22:09:53.908 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 158.9 0.00 0.00 1 0.000 0 0 2024-09-20T22:09:53.908 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: 2024-09-20T22:09:53.908 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: ** Compaction Stats [default] ** 2024-09-20T22:09:53.908 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-20T22:09:53.909 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-20T22:09:53.909 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 158.9 0.00 0.00 1 0.000 0 0 2024-09-20T22:09:53.909 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-20T22:09:53.909 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-20T22:09:53.909 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-20T22:09:53.909 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-20T22:09:53.909 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-20T22:09:53.909 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-20T22:09:53.909 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: Cumulative compaction: 0.00 GB write, 25.02 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-20T22:09:53.909 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: Interval compaction: 0.00 GB write, 25.02 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-20T22:09:53.909 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-20T22:09:53.909 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: 2024-09-20T22:09:53.909 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-20T22:09:53.910 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: 2024-09-20T22:09:53.910 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: ** Compaction Stats [default] ** 2024-09-20T22:09:53.910 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-20T22:09:53.910 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-20T22:09:53.910 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: L0 2/0 79.75 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 158.9 0.00 0.00 1 0.000 0 0 2024-09-20T22:09:53.910 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: Sum 2/0 79.75 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 158.9 0.00 0.00 1 0.000 0 0 2024-09-20T22:09:53.910 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-09-20T22:09:53.910 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: 2024-09-20T22:09:53.910 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: ** Compaction Stats [default] ** 2024-09-20T22:09:53.910 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-20T22:09:53.910 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-20T22:09:53.910 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 158.9 0.00 0.00 1 0.000 0 0 2024-09-20T22:09:53.910 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-20T22:09:53.910 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-20T22:09:53.911 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-20T22:09:53.911 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-20T22:09:53.911 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-20T22:09:53.911 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-20T22:09:53.911 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: Cumulative compaction: 0.00 GB write, 24.77 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-20T22:09:53.911 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-20T22:09:53.911 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-20T22:09:53.911 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: 2024-09-20T22:09:53.911 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-20T22:09:53.911 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: starting mon.smithi037 rank 0 at public addrs [v2:172.21.15.37:3300/0,v1:172.21.15.37:6789/0] at bind addrs [v2:172.21.15.37:3300/0,v1:172.21.15.37:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi037 fsid f9a6357a-779c-11ef-baf5-efdc52797490 2024-09-20T22:09:53.911 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: mon.smithi037@-1(???) e1 preinit fsid f9a6357a-779c-11ef-baf5-efdc52797490 2024-09-20T22:09:53.911 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: mon.smithi037@-1(???).mds e1 new map 2024-09-20T22:09:53.911 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: mon.smithi037@-1(???).mds e1 print_map 2024-09-20T22:09:53.911 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: e1 2024-09-20T22:09:53.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-20T22:09:53.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-20T22:09:53.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: legacy client fscid: -1 2024-09-20T22:09:53.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: 2024-09-20T22:09:53.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: No filesystems configured 2024-09-20T22:09:53.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: mon.smithi037@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-09-20T22:09:53.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: mon.smithi037@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-20T22:09:53.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: mon.smithi037@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-20T22:09:53.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: mon.smithi037@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-20T22:09:53.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: mon.smithi037@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-09-20T22:09:53.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 bash[23699]: 71f591122786ae60b80a25133890317cf9cad97c3c7ba98976cff6c0bb977f18 2024-09-20T22:09:53.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 systemd[1]: Started Ceph mon.smithi037 for f9a6357a-779c-11ef-baf5-efdc52797490. 2024-09-20T22:09:53.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: mon.smithi037 is new leader, mons smithi037 in quorum (ranks 0) 2024-09-20T22:09:53.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: monmap e1: 1 mons at {smithi037=[v2:172.21.15.37:3300/0,v1:172.21.15.37:6789/0]} 2024-09-20T22:09:53.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: fsmap 2024-09-20T22:09:53.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: osdmap e1: 0 total, 0 up, 0 in 2024-09-20T22:09:53.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:53 smithi037 ceph-mon[23718]: mgrmap e1: no daemons active 2024-09-20T22:09:55.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:55 smithi037 ceph-mon[23718]: from='client.? 172.21.15.37:0/2737792583' entity='client.admin' 2024-09-20T22:09:55.778 INFO:teuthology.orchestra.run.smithi037.stderr:Wrote config to /etc/ceph/ceph.conf 2024-09-20T22:09:55.778 INFO:teuthology.orchestra.run.smithi037.stderr:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-09-20T22:09:55.778 INFO:teuthology.orchestra.run.smithi037.stderr:Creating mgr... 2024-09-20T22:09:55.779 INFO:teuthology.orchestra.run.smithi037.stderr:Verifying port 9283 ... 2024-09-20T22:09:56.114 INFO:teuthology.orchestra.run.smithi037.stderr:systemctl: Failed to reset failed state of unit ceph-f9a6357a-779c-11ef-baf5-efdc52797490@mgr.smithi037.lykblh.service: Unit ceph-f9a6357a-779c-11ef-baf5-efdc52797490@mgr.smithi037.lykblh.service not loaded. 2024-09-20T22:09:56.123 INFO:teuthology.orchestra.run.smithi037.stderr:systemctl: Created symlink /etc/systemd/system/ceph-f9a6357a-779c-11ef-baf5-efdc52797490.target.wants/ceph-f9a6357a-779c-11ef-baf5-efdc52797490@mgr.smithi037.lykblh.service → /etc/systemd/system/ceph-f9a6357a-779c-11ef-baf5-efdc52797490@.service. 2024-09-20T22:09:56.983 INFO:teuthology.orchestra.run.smithi037.stderr:systemctl: disabled 2024-09-20T22:09:56.991 INFO:teuthology.orchestra.run.smithi037.stderr:systemctl: inactive 2024-09-20T22:09:56.991 INFO:teuthology.orchestra.run.smithi037.stderr:firewalld.service is not enabled 2024-09-20T22:09:56.991 INFO:teuthology.orchestra.run.smithi037.stderr:Not possible to enable service . firewalld.service is not available 2024-09-20T22:09:57.000 INFO:teuthology.orchestra.run.smithi037.stderr:systemctl: disabled 2024-09-20T22:09:57.007 INFO:teuthology.orchestra.run.smithi037.stderr:systemctl: inactive 2024-09-20T22:09:57.008 INFO:teuthology.orchestra.run.smithi037.stderr:firewalld.service is not enabled 2024-09-20T22:09:57.008 INFO:teuthology.orchestra.run.smithi037.stderr:Not possible to open ports <[9283]>. firewalld.service is not available 2024-09-20T22:09:57.008 INFO:teuthology.orchestra.run.smithi037.stderr:Waiting for mgr to start... 2024-09-20T22:09:57.008 INFO:teuthology.orchestra.run.smithi037.stderr:Waiting for mgr... 2024-09-20T22:09:57.559 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: 2024-09-20T22:09:57.559 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: { 2024-09-20T22:09:57.559 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "fsid": "f9a6357a-779c-11ef-baf5-efdc52797490", 2024-09-20T22:09:57.559 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "health": { 2024-09-20T22:09:57.559 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-20T22:09:57.559 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "checks": {}, 2024-09-20T22:09:57.559 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "mutes": [] 2024-09-20T22:09:57.559 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: }, 2024-09-20T22:09:57.559 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-20T22:09:57.559 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "quorum": [ 2024-09-20T22:09:57.559 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: 0 2024-09-20T22:09:57.560 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: ], 2024-09-20T22:09:57.560 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-20T22:09:57.560 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "smithi037" 2024-09-20T22:09:57.560 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: ], 2024-09-20T22:09:57.560 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "quorum_age": 3, 2024-09-20T22:09:57.560 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "monmap": { 2024-09-20T22:09:57.560 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-20T22:09:57.560 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-20T22:09:57.560 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-20T22:09:57.560 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: }, 2024-09-20T22:09:57.560 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "osdmap": { 2024-09-20T22:09:57.560 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-20T22:09:57.560 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-20T22:09:57.560 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-20T22:09:57.560 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-20T22:09:57.561 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-20T22:09:57.561 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-20T22:09:57.563 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-20T22:09:57.564 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: }, 2024-09-20T22:09:57.564 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "pgmap": { 2024-09-20T22:09:57.564 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-20T22:09:57.564 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-20T22:09:57.564 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-20T22:09:57.564 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-20T22:09:57.564 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-20T22:09:57.564 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-20T22:09:57.564 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-20T22:09:57.564 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-20T22:09:57.564 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: }, 2024-09-20T22:09:57.564 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "fsmap": { 2024-09-20T22:09:57.564 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-20T22:09:57.564 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-20T22:09:57.565 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-20T22:09:57.565 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: }, 2024-09-20T22:09:57.565 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-20T22:09:57.565 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "available": false, 2024-09-20T22:09:57.565 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-20T22:09:57.565 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "modules": [ 2024-09-20T22:09:57.565 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "iostat", 2024-09-20T22:09:57.565 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "nfs", 2024-09-20T22:09:57.565 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "restful" 2024-09-20T22:09:57.565 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: ], 2024-09-20T22:09:57.565 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "services": {} 2024-09-20T22:09:57.565 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: }, 2024-09-20T22:09:57.565 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "servicemap": { 2024-09-20T22:09:57.565 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-20T22:09:57.565 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "modified": "2024-09-20T22:09:43.123122+0000", 2024-09-20T22:09:57.565 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "services": {} 2024-09-20T22:09:57.566 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: }, 2024-09-20T22:09:57.566 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-20T22:09:57.566 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: } 2024-09-20T22:09:57.867 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:09:57 smithi037 ceph-mon[23718]: from='client.? 172.21.15.37:0/2305100218' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-20T22:09:58.179 INFO:teuthology.orchestra.run.smithi037.stderr:mgr not available, waiting (1/15)... 2024-09-20T22:10:00.673 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: 2024-09-20T22:10:00.673 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: { 2024-09-20T22:10:00.673 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "fsid": "f9a6357a-779c-11ef-baf5-efdc52797490", 2024-09-20T22:10:00.673 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "health": { 2024-09-20T22:10:00.673 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-20T22:10:00.673 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "checks": {}, 2024-09-20T22:10:00.673 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "mutes": [] 2024-09-20T22:10:00.673 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: }, 2024-09-20T22:10:00.673 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-20T22:10:00.674 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "quorum": [ 2024-09-20T22:10:00.674 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: 0 2024-09-20T22:10:00.674 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: ], 2024-09-20T22:10:00.674 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-20T22:10:00.674 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "smithi037" 2024-09-20T22:10:00.674 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: ], 2024-09-20T22:10:00.674 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "quorum_age": 7, 2024-09-20T22:10:00.674 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "monmap": { 2024-09-20T22:10:00.674 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-20T22:10:00.674 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-20T22:10:00.674 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-20T22:10:00.674 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: }, 2024-09-20T22:10:00.674 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "osdmap": { 2024-09-20T22:10:00.675 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-20T22:10:00.675 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-20T22:10:00.675 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-20T22:10:00.675 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-20T22:10:00.675 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-20T22:10:00.675 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-20T22:10:00.675 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-20T22:10:00.675 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: }, 2024-09-20T22:10:00.675 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "pgmap": { 2024-09-20T22:10:00.675 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-20T22:10:00.675 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-20T22:10:00.675 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-20T22:10:00.675 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-20T22:10:00.675 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-20T22:10:00.675 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-20T22:10:00.676 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-20T22:10:00.676 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-20T22:10:00.676 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: }, 2024-09-20T22:10:00.676 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "fsmap": { 2024-09-20T22:10:00.676 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-20T22:10:00.676 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-20T22:10:00.676 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-20T22:10:00.676 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: }, 2024-09-20T22:10:00.676 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-20T22:10:00.676 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "available": false, 2024-09-20T22:10:00.676 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-20T22:10:00.676 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "modules": [ 2024-09-20T22:10:00.676 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "iostat", 2024-09-20T22:10:00.676 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "nfs", 2024-09-20T22:10:00.676 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "restful" 2024-09-20T22:10:00.677 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: ], 2024-09-20T22:10:00.677 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "services": {} 2024-09-20T22:10:00.677 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: }, 2024-09-20T22:10:00.677 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "servicemap": { 2024-09-20T22:10:00.677 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-20T22:10:00.677 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "modified": "2024-09-20T22:09:43.123122+0000", 2024-09-20T22:10:00.677 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "services": {} 2024-09-20T22:10:00.677 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: }, 2024-09-20T22:10:00.677 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-20T22:10:00.677 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: } 2024-09-20T22:10:00.899 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:00 smithi037 ceph-mon[23718]: from='client.? 172.21.15.37:0/1310091734' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-20T22:10:01.336 INFO:teuthology.orchestra.run.smithi037.stderr:mgr not available, waiting (2/15)... 2024-09-20T22:10:03.580 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:03 smithi037 ceph-mon[23718]: Activating manager daemon smithi037.lykblh 2024-09-20T22:10:03.581 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:03 smithi037 ceph-mon[23718]: mgrmap e2: smithi037.lykblh(active, starting, since 0.00269774s) 2024-09-20T22:10:03.581 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:03 smithi037 ceph-mon[23718]: from='mgr.14100 172.21.15.37:0/4076620482' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-20T22:10:03.581 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:03 smithi037 ceph-mon[23718]: from='mgr.14100 172.21.15.37:0/4076620482' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-20T22:10:03.581 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:03 smithi037 ceph-mon[23718]: from='mgr.14100 172.21.15.37:0/4076620482' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-20T22:10:03.581 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:03 smithi037 ceph-mon[23718]: from='mgr.14100 172.21.15.37:0/4076620482' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mon metadata", "id": "smithi037"}]: dispatch 2024-09-20T22:10:03.581 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:03 smithi037 ceph-mon[23718]: from='mgr.14100 172.21.15.37:0/4076620482' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mgr metadata", "who": "smithi037.lykblh", "id": "smithi037.lykblh"}]: dispatch 2024-09-20T22:10:03.581 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:03 smithi037 ceph-mon[23718]: Manager daemon smithi037.lykblh is now available 2024-09-20T22:10:03.581 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:03 smithi037 ceph-mon[23718]: from='mgr.14100 172.21.15.37:0/4076620482' entity='mgr.smithi037.lykblh' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi037.lykblh/mirror_snapshot_schedule"}]: dispatch 2024-09-20T22:10:03.581 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:03 smithi037 ceph-mon[23718]: from='mgr.14100 172.21.15.37:0/4076620482' entity='mgr.smithi037.lykblh' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi037.lykblh/trash_purge_schedule"}]: dispatch 2024-09-20T22:10:03.581 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:03 smithi037 ceph-mon[23718]: from='mgr.14100 172.21.15.37:0/4076620482' entity='mgr.smithi037.lykblh' 2024-09-20T22:10:03.581 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:03 smithi037 ceph-mon[23718]: from='mgr.14100 172.21.15.37:0/4076620482' entity='mgr.smithi037.lykblh' 2024-09-20T22:10:03.582 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:03 smithi037 ceph-mon[23718]: from='mgr.14100 172.21.15.37:0/4076620482' entity='mgr.smithi037.lykblh' 2024-09-20T22:10:03.884 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: 2024-09-20T22:10:03.884 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: { 2024-09-20T22:10:03.884 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "fsid": "f9a6357a-779c-11ef-baf5-efdc52797490", 2024-09-20T22:10:03.884 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "health": { 2024-09-20T22:10:03.884 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-20T22:10:03.884 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "checks": {}, 2024-09-20T22:10:03.884 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "mutes": [] 2024-09-20T22:10:03.884 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: }, 2024-09-20T22:10:03.884 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-20T22:10:03.884 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "quorum": [ 2024-09-20T22:10:03.884 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: 0 2024-09-20T22:10:03.885 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: ], 2024-09-20T22:10:03.885 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-20T22:10:03.885 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "smithi037" 2024-09-20T22:10:03.885 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: ], 2024-09-20T22:10:03.885 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "quorum_age": 10, 2024-09-20T22:10:03.885 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "monmap": { 2024-09-20T22:10:03.885 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-20T22:10:03.885 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-20T22:10:03.885 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-20T22:10:03.885 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: }, 2024-09-20T22:10:03.885 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "osdmap": { 2024-09-20T22:10:03.885 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-20T22:10:03.885 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-20T22:10:03.885 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-20T22:10:03.885 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-20T22:10:03.886 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-20T22:10:03.888 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-20T22:10:03.889 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-20T22:10:03.889 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: }, 2024-09-20T22:10:03.889 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "pgmap": { 2024-09-20T22:10:03.889 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-20T22:10:03.889 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-20T22:10:03.889 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-20T22:10:03.889 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-20T22:10:03.889 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-20T22:10:03.889 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-20T22:10:03.889 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-20T22:10:03.889 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-20T22:10:03.889 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: }, 2024-09-20T22:10:03.889 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "fsmap": { 2024-09-20T22:10:03.889 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-20T22:10:03.890 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-20T22:10:03.890 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-20T22:10:03.890 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: }, 2024-09-20T22:10:03.890 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-20T22:10:03.890 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "available": false, 2024-09-20T22:10:03.890 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-20T22:10:03.890 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "modules": [ 2024-09-20T22:10:03.890 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "iostat", 2024-09-20T22:10:03.890 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "nfs", 2024-09-20T22:10:03.890 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "restful" 2024-09-20T22:10:03.890 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: ], 2024-09-20T22:10:03.890 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "services": {} 2024-09-20T22:10:03.890 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: }, 2024-09-20T22:10:03.890 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "servicemap": { 2024-09-20T22:10:03.890 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-20T22:10:03.891 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "modified": "2024-09-20T22:09:43.123122+0000", 2024-09-20T22:10:03.891 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "services": {} 2024-09-20T22:10:03.891 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: }, 2024-09-20T22:10:03.891 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-20T22:10:03.891 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: } 2024-09-20T22:10:04.500 INFO:teuthology.orchestra.run.smithi037.stderr:mgr not available, waiting (3/15)... 2024-09-20T22:10:04.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:04 smithi037 ceph-mon[23718]: from='client.? 172.21.15.37:0/2503556867' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-20T22:10:04.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:04 smithi037 ceph-mon[23718]: mgrmap e3: smithi037.lykblh(active, since 1.00617s) 2024-09-20T22:10:06.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:06 smithi037 ceph-mon[23718]: mgrmap e4: smithi037.lykblh(active, since 2s) 2024-09-20T22:10:07.204 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: 2024-09-20T22:10:07.205 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: { 2024-09-20T22:10:07.205 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "fsid": "f9a6357a-779c-11ef-baf5-efdc52797490", 2024-09-20T22:10:07.205 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "health": { 2024-09-20T22:10:07.205 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-20T22:10:07.205 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "checks": {}, 2024-09-20T22:10:07.205 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "mutes": [] 2024-09-20T22:10:07.205 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: }, 2024-09-20T22:10:07.206 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-20T22:10:07.206 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "quorum": [ 2024-09-20T22:10:07.206 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: 0 2024-09-20T22:10:07.206 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: ], 2024-09-20T22:10:07.206 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-20T22:10:07.206 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "smithi037" 2024-09-20T22:10:07.206 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: ], 2024-09-20T22:10:07.206 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "quorum_age": 13, 2024-09-20T22:10:07.206 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "monmap": { 2024-09-20T22:10:07.206 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-20T22:10:07.206 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-20T22:10:07.207 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-20T22:10:07.207 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: }, 2024-09-20T22:10:07.207 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "osdmap": { 2024-09-20T22:10:07.207 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-20T22:10:07.207 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-20T22:10:07.207 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-20T22:10:07.207 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-20T22:10:07.207 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-20T22:10:07.207 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-20T22:10:07.207 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-20T22:10:07.207 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: }, 2024-09-20T22:10:07.207 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "pgmap": { 2024-09-20T22:10:07.207 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-20T22:10:07.207 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-20T22:10:07.207 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-20T22:10:07.208 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-20T22:10:07.208 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-20T22:10:07.208 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-20T22:10:07.208 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-20T22:10:07.208 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-20T22:10:07.208 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: }, 2024-09-20T22:10:07.208 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "fsmap": { 2024-09-20T22:10:07.208 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-20T22:10:07.208 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-20T22:10:07.208 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-20T22:10:07.208 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: }, 2024-09-20T22:10:07.208 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-20T22:10:07.208 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "available": true, 2024-09-20T22:10:07.208 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-20T22:10:07.208 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "modules": [ 2024-09-20T22:10:07.209 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "iostat", 2024-09-20T22:10:07.209 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "nfs", 2024-09-20T22:10:07.209 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "restful" 2024-09-20T22:10:07.209 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: ], 2024-09-20T22:10:07.209 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "services": {} 2024-09-20T22:10:07.209 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: }, 2024-09-20T22:10:07.209 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "servicemap": { 2024-09-20T22:10:07.209 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-20T22:10:07.209 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "modified": "2024-09-20T22:09:43.123122+0000", 2024-09-20T22:10:07.209 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "services": {} 2024-09-20T22:10:07.209 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: }, 2024-09-20T22:10:07.209 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-20T22:10:07.209 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: } 2024-09-20T22:10:07.806 INFO:teuthology.orchestra.run.smithi037.stderr:mgr is available 2024-09-20T22:10:07.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:07 smithi037 ceph-mon[23718]: from='client.? 172.21.15.37:0/3687684486' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-20T22:10:08.402 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: 2024-09-20T22:10:08.402 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: [global] 2024-09-20T22:10:08.402 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: fsid = f9a6357a-779c-11ef-baf5-efdc52797490 2024-09-20T22:10:08.403 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-09-20T22:10:08.403 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-09-20T22:10:08.403 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-09-20T22:10:08.403 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-09-20T22:10:08.403 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: 2024-09-20T22:10:08.403 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: [mgr] 2024-09-20T22:10:08.403 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-09-20T22:10:08.403 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: 2024-09-20T22:10:08.403 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: [osd] 2024-09-20T22:10:08.403 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-09-20T22:10:08.403 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-09-20T22:10:08.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:08 smithi037 ceph-mon[23718]: from='client.? 172.21.15.37:0/741233899' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-09-20T22:10:08.978 INFO:teuthology.orchestra.run.smithi037.stderr:Enabling cephadm module... 2024-09-20T22:10:09.972 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:09 smithi037 ceph-mon[23718]: from='client.? 172.21.15.37:0/4255616899' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-09-20T22:10:10.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:10 smithi037 ceph-mon[23718]: from='client.? 172.21.15.37:0/4255616899' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-09-20T22:10:10.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:10 smithi037 ceph-mon[23718]: mgrmap e5: smithi037.lykblh(active, since 6s) 2024-09-20T22:10:10.956 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: { 2024-09-20T22:10:10.956 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "epoch": 5, 2024-09-20T22:10:10.956 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "available": true, 2024-09-20T22:10:10.956 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "active_name": "smithi037.lykblh", 2024-09-20T22:10:10.956 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "num_standby": 0 2024-09-20T22:10:10.956 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: } 2024-09-20T22:10:11.779 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:11 smithi037 ceph-mon[23718]: from='client.? 172.21.15.37:0/2776087240' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-09-20T22:10:11.782 INFO:teuthology.orchestra.run.smithi037.stderr:Waiting for the mgr to restart... 2024-09-20T22:10:11.782 INFO:teuthology.orchestra.run.smithi037.stderr:Waiting for mgr epoch 5... 2024-09-20T22:10:16.412 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:16 smithi037 ceph-mon[23718]: Active manager daemon smithi037.lykblh restarted 2024-09-20T22:10:16.412 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:16 smithi037 ceph-mon[23718]: Activating manager daemon smithi037.lykblh 2024-09-20T22:10:16.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:16 smithi037 ceph-mon[23718]: osdmap e2: 0 total, 0 up, 0 in 2024-09-20T22:10:17.138 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: { 2024-09-20T22:10:17.139 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "mgrmap_epoch": 7, 2024-09-20T22:10:17.139 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "initialized": true 2024-09-20T22:10:17.139 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: } 2024-09-20T22:10:17.412 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:17 smithi037 ceph-mon[23718]: mgrmap e6: smithi037.lykblh(active, starting, since 0.0522934s) 2024-09-20T22:10:17.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:17 smithi037 ceph-mon[23718]: from='mgr.14120 172.21.15.37:0/2069621640' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mon metadata", "id": "smithi037"}]: dispatch 2024-09-20T22:10:17.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:17 smithi037 ceph-mon[23718]: from='mgr.14120 172.21.15.37:0/2069621640' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mgr metadata", "who": "smithi037.lykblh", "id": "smithi037.lykblh"}]: dispatch 2024-09-20T22:10:17.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:17 smithi037 ceph-mon[23718]: from='mgr.14120 172.21.15.37:0/2069621640' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-20T22:10:17.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:17 smithi037 ceph-mon[23718]: from='mgr.14120 172.21.15.37:0/2069621640' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-20T22:10:17.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:17 smithi037 ceph-mon[23718]: from='mgr.14120 172.21.15.37:0/2069621640' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-20T22:10:17.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:17 smithi037 ceph-mon[23718]: Manager daemon smithi037.lykblh is now available 2024-09-20T22:10:17.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:17 smithi037 ceph-mon[23718]: from='mgr.14120 172.21.15.37:0/2069621640' entity='mgr.smithi037.lykblh' 2024-09-20T22:10:17.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:17 smithi037 ceph-mon[23718]: from='mgr.14120 172.21.15.37:0/2069621640' entity='mgr.smithi037.lykblh' 2024-09-20T22:10:17.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:17 smithi037 ceph-mon[23718]: from='mgr.14120 172.21.15.37:0/2069621640' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:10:17.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:17 smithi037 ceph-mon[23718]: from='mgr.14120 172.21.15.37:0/2069621640' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:10:17.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:17 smithi037 ceph-mon[23718]: from='mgr.14120 172.21.15.37:0/2069621640' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:10:17.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:17 smithi037 ceph-mon[23718]: from='mgr.14120 172.21.15.37:0/2069621640' entity='mgr.smithi037.lykblh' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi037.lykblh/mirror_snapshot_schedule"}]: dispatch 2024-09-20T22:10:17.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:17 smithi037 ceph-mon[23718]: from='mgr.14120 172.21.15.37:0/2069621640' entity='mgr.smithi037.lykblh' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi037.lykblh/trash_purge_schedule"}]: dispatch 2024-09-20T22:10:17.721 INFO:teuthology.orchestra.run.smithi037.stderr:mgr epoch 5 is available 2024-09-20T22:10:17.721 INFO:teuthology.orchestra.run.smithi037.stderr:Setting orchestrator backend to cephadm... 2024-09-20T22:10:18.347 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:18 smithi037 ceph-mon[23718]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-09-20T22:10:18.348 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:18 smithi037 ceph-mon[23718]: mgrmap e7: smithi037.lykblh(active, since 1.05944s) 2024-09-20T22:10:18.348 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:18 smithi037 ceph-mon[23718]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-09-20T22:10:18.348 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:18 smithi037 ceph-mon[23718]: from='mgr.14120 172.21.15.37:0/2069621640' entity='mgr.smithi037.lykblh' 2024-09-20T22:10:18.348 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:18 smithi037 ceph-mon[23718]: from='mgr.14120 172.21.15.37:0/2069621640' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:10:19.608 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:19 smithi037 ceph-mon[23718]: [20/Sep/2024:22:10:17] ENGINE Bus STARTING 2024-09-20T22:10:19.608 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:19 smithi037 ceph-mon[23718]: [20/Sep/2024:22:10:17] ENGINE Serving on https://172.21.15.37:7150 2024-09-20T22:10:19.608 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:19 smithi037 ceph-mon[23718]: [20/Sep/2024:22:10:17] ENGINE Bus STARTED 2024-09-20T22:10:19.608 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:19 smithi037 ceph-mon[23718]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:10:19.608 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:19 smithi037 ceph-mon[23718]: from='mgr.14120 172.21.15.37:0/2069621640' entity='mgr.smithi037.lykblh' 2024-09-20T22:10:19.608 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:19 smithi037 ceph-mon[23718]: from='mgr.14120 172.21.15.37:0/2069621640' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:10:19.608 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:19 smithi037 ceph-mon[23718]: mgrmap e8: smithi037.lykblh(active, since 2s) 2024-09-20T22:10:19.608 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: value unchanged 2024-09-20T22:10:20.200 INFO:teuthology.orchestra.run.smithi037.stderr:Generating ssh key... 2024-09-20T22:10:20.632 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:20 smithi037 ceph-mon[23718]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:10:22.869 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:22 smithi037 ceph-mon[23718]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:10:22.869 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:22 smithi037 ceph-mon[23718]: Generating ssh key... 2024-09-20T22:10:22.869 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:22 smithi037 ceph-mon[23718]: from='mgr.14120 172.21.15.37:0/2069621640' entity='mgr.smithi037.lykblh' 2024-09-20T22:10:22.869 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:22 smithi037 ceph-mon[23718]: from='mgr.14120 172.21.15.37:0/2069621640' entity='mgr.smithi037.lykblh' 2024-09-20T22:10:22.869 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCvwq6Bv1SrRrqqnGsaspxqsm7ELTL2SSveARgkraAk5Om/8TMKeQKiW4PdHopgBggVe2e0pPbmNRc7lCD/manhAsEz8pD79ITydPGuBd1YzAE1WU80cxNO5Jgh7I+/FqQK9alwJxoEqEg1FRInU+Jy7vK1NJbN8+lZYSSbA7WeuBnV277TMGTByrOioFYHbCCPHClaCCVfO/af9HpWJW+JOmrpm5PHOAlTaZyFXFOvbGMyjOs+E2dNJgvtjpQOcwxrYrpln/Dk2KO6/ICXlxaQr57N2DNzQG+KXrl7B+1IKHgOgCez9wG9vKLR+CYxiniaxQcP5M1i/lFOCvxBEYg2GOqdmYJ7dPzRGasSvv92+mFkWMewmHsriieo/IH+9akLgcjy+DX/8gfxASzb8D8+B8vU7IrKDYIBFo7Eo0IWpRU0dQTwGBSYo8uHjoluGWRGrwgTlbSEzbKy4gGiTLXjNYlaVqdKaN9A30wJdQbCimaLIT9ZBIvP07hMYUhpDGc= ceph-f9a6357a-779c-11ef-baf5-efdc52797490 2024-09-20T22:10:23.437 INFO:teuthology.orchestra.run.smithi037.stderr:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-09-20T22:10:23.437 INFO:teuthology.orchestra.run.smithi037.stderr:Adding key to root@localhost authorized_keys... 2024-09-20T22:10:23.443 INFO:teuthology.orchestra.run.smithi037.stderr:Adding host smithi037... 2024-09-20T22:10:24.662 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:24 smithi037 ceph-mon[23718]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:10:24.989 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: Added host 'smithi037' with addr '172.21.15.37' 2024-09-20T22:10:25.618 INFO:teuthology.orchestra.run.smithi037.stderr:Deploying mon service with default placement... 2024-09-20T22:10:25.857 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:25 smithi037 ceph-mon[23718]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi037", "addr": "172.21.15.37", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:10:25.858 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:25 smithi037 ceph-mon[23718]: Deploying cephadm binary to smithi037 2024-09-20T22:10:25.858 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:25 smithi037 ceph-mon[23718]: from='mgr.14120 172.21.15.37:0/2069621640' entity='mgr.smithi037.lykblh' 2024-09-20T22:10:25.858 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:25 smithi037 ceph-mon[23718]: from='mgr.14120 172.21.15.37:0/2069621640' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:10:26.301 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: Scheduled mon update... 2024-09-20T22:10:26.649 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:26 smithi037 ceph-mon[23718]: Added host smithi037 2024-09-20T22:10:26.649 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:26 smithi037 ceph-mon[23718]: from='mgr.14120 172.21.15.37:0/2069621640' entity='mgr.smithi037.lykblh' 2024-09-20T22:10:27.764 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:27 smithi037 ceph-mon[23718]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:10:27.764 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:27 smithi037 ceph-mon[23718]: Saving service mon spec with placement count:5 2024-09-20T22:10:27.764 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:27 smithi037 ceph-mon[23718]: from='mgr.14120 172.21.15.37:0/2069621640' entity='mgr.smithi037.lykblh' 2024-09-20T22:10:27.766 INFO:teuthology.orchestra.run.smithi037.stderr:Deploying mgr service with default placement... 2024-09-20T22:10:28.374 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: Scheduled mgr update... 2024-09-20T22:10:28.984 INFO:teuthology.orchestra.run.smithi037.stderr:Deploying crash service with default placement... 2024-09-20T22:10:29.662 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:29 smithi037 ceph-mon[23718]: from='mgr.14120 172.21.15.37:0/2069621640' entity='mgr.smithi037.lykblh' 2024-09-20T22:10:29.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:29 smithi037 ceph-mon[23718]: from='client.14144 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:10:29.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:29 smithi037 ceph-mon[23718]: Saving service mgr spec with placement count:2 2024-09-20T22:10:29.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:29 smithi037 ceph-mon[23718]: from='mgr.14120 172.21.15.37:0/2069621640' entity='mgr.smithi037.lykblh' 2024-09-20T22:10:29.672 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: Scheduled crash update... 2024-09-20T22:10:30.768 INFO:teuthology.orchestra.run.smithi037.stderr:Deploying prometheus service with default placement... 2024-09-20T22:10:30.946 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:30 smithi037 ceph-mon[23718]: from='client.14146 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "crash", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:10:30.946 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:30 smithi037 ceph-mon[23718]: Saving service crash spec with placement * 2024-09-20T22:10:30.946 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:30 smithi037 ceph-mon[23718]: from='mgr.14120 172.21.15.37:0/2069621640' entity='mgr.smithi037.lykblh' 2024-09-20T22:10:30.946 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:30 smithi037 ceph-mon[23718]: from='mgr.14120 172.21.15.37:0/2069621640' entity='mgr.smithi037.lykblh' 2024-09-20T22:10:31.445 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: Scheduled prometheus update... 2024-09-20T22:10:32.016 INFO:teuthology.orchestra.run.smithi037.stderr:Deploying grafana service with default placement... 2024-09-20T22:10:32.594 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:32 smithi037 ceph-mon[23718]: from='mgr.14120 172.21.15.37:0/2069621640' entity='mgr.smithi037.lykblh' 2024-09-20T22:10:32.594 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:32 smithi037 ceph-mon[23718]: from='client.14148 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:10:32.594 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:32 smithi037 ceph-mon[23718]: Saving service prometheus spec with placement count:1 2024-09-20T22:10:32.594 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:32 smithi037 ceph-mon[23718]: from='mgr.14120 172.21.15.37:0/2069621640' entity='mgr.smithi037.lykblh' 2024-09-20T22:10:32.695 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: Scheduled grafana update... 2024-09-20T22:10:33.726 INFO:teuthology.orchestra.run.smithi037.stderr:Deploying node-exporter service with default placement... 2024-09-20T22:10:33.849 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:33 smithi037 ceph-mon[23718]: from='client.14150 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:10:33.849 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:33 smithi037 ceph-mon[23718]: Saving service grafana spec with placement count:1 2024-09-20T22:10:33.849 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:33 smithi037 ceph-mon[23718]: from='mgr.14120 172.21.15.37:0/2069621640' entity='mgr.smithi037.lykblh' 2024-09-20T22:10:34.368 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: Scheduled node-exporter update... 2024-09-20T22:10:35.159 INFO:teuthology.orchestra.run.smithi037.stderr:Deploying alertmanager service with default placement... 2024-09-20T22:10:35.412 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:35 smithi037 ceph-mon[23718]: from='client.14152 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:10:35.412 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:35 smithi037 ceph-mon[23718]: Saving service node-exporter spec with placement * 2024-09-20T22:10:35.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:35 smithi037 ceph-mon[23718]: from='mgr.14120 172.21.15.37:0/2069621640' entity='mgr.smithi037.lykblh' 2024-09-20T22:10:35.837 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: Scheduled alertmanager update... 2024-09-20T22:10:37.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:36 smithi037 ceph-mon[23718]: from='client.14154 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:10:37.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:36 smithi037 ceph-mon[23718]: Saving service alertmanager spec with placement count:1 2024-09-20T22:10:37.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:36 smithi037 ceph-mon[23718]: from='mgr.14120 172.21.15.37:0/2069621640' entity='mgr.smithi037.lykblh' 2024-09-20T22:10:37.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:36 smithi037 ceph-mon[23718]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T22:10:38.412 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:38 smithi037 ceph-mon[23718]: from='client.? 172.21.15.37:0/3385100023' entity='client.admin' 2024-09-20T22:10:39.030 INFO:teuthology.orchestra.run.smithi037.stderr:Enabling the dashboard module... 2024-09-20T22:10:39.662 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:39 smithi037 ceph-mon[23718]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T22:10:39.662 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:39 smithi037 ceph-mon[23718]: from='client.? 172.21.15.37:0/693229635' entity='client.admin' 2024-09-20T22:10:40.707 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:40 smithi037 ceph-mon[23718]: from='client.? 172.21.15.37:0/86145625' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-09-20T22:10:41.577 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:41 smithi037 ceph-mon[23718]: from='client.? 172.21.15.37:0/86145625' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-09-20T22:10:41.577 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:41 smithi037 ceph-mon[23718]: mgrmap e9: smithi037.lykblh(active, since 24s) 2024-09-20T22:10:41.835 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: { 2024-09-20T22:10:41.835 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "epoch": 9, 2024-09-20T22:10:41.835 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "available": true, 2024-09-20T22:10:41.835 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "active_name": "smithi037.lykblh", 2024-09-20T22:10:41.835 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "num_standby": 0 2024-09-20T22:10:41.835 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: } 2024-09-20T22:10:42.623 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:42 smithi037 ceph-mon[23718]: from='client.? 172.21.15.37:0/2853524224' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-09-20T22:10:42.625 INFO:teuthology.orchestra.run.smithi037.stderr:Waiting for the mgr to restart... 2024-09-20T22:10:42.625 INFO:teuthology.orchestra.run.smithi037.stderr:Waiting for mgr epoch 9... 2024-09-20T22:10:47.865 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: { 2024-09-20T22:10:47.866 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "mgrmap_epoch": 11, 2024-09-20T22:10:47.866 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: "initialized": true 2024-09-20T22:10:47.866 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: } 2024-09-20T22:10:47.980 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:47 smithi037 ceph-mon[23718]: Active manager daemon smithi037.lykblh restarted 2024-09-20T22:10:47.980 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:47 smithi037 ceph-mon[23718]: Activating manager daemon smithi037.lykblh 2024-09-20T22:10:47.980 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:47 smithi037 ceph-mon[23718]: osdmap e3: 0 total, 0 up, 0 in 2024-09-20T22:10:47.980 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:47 smithi037 ceph-mon[23718]: mgrmap e10: smithi037.lykblh(active, starting, since 0.05368s) 2024-09-20T22:10:47.981 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:47 smithi037 ceph-mon[23718]: from='mgr.14162 172.21.15.37:0/1762807819' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mon metadata", "id": "smithi037"}]: dispatch 2024-09-20T22:10:47.981 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:47 smithi037 ceph-mon[23718]: from='mgr.14162 172.21.15.37:0/1762807819' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mgr metadata", "who": "smithi037.lykblh", "id": "smithi037.lykblh"}]: dispatch 2024-09-20T22:10:47.981 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:47 smithi037 ceph-mon[23718]: from='mgr.14162 172.21.15.37:0/1762807819' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-20T22:10:47.981 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:47 smithi037 ceph-mon[23718]: from='mgr.14162 172.21.15.37:0/1762807819' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-20T22:10:47.981 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:47 smithi037 ceph-mon[23718]: from='mgr.14162 172.21.15.37:0/1762807819' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-20T22:10:47.981 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:47 smithi037 ceph-mon[23718]: Manager daemon smithi037.lykblh is now available 2024-09-20T22:10:47.981 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:47 smithi037 ceph-mon[23718]: from='mgr.14162 172.21.15.37:0/1762807819' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:10:47.981 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:47 smithi037 ceph-mon[23718]: from='mgr.14162 172.21.15.37:0/1762807819' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:10:47.981 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:47 smithi037 ceph-mon[23718]: from='mgr.14162 172.21.15.37:0/1762807819' entity='mgr.smithi037.lykblh' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi037.lykblh/mirror_snapshot_schedule"}]: dispatch 2024-09-20T22:10:47.981 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:47 smithi037 ceph-mon[23718]: from='mgr.14162 172.21.15.37:0/1762807819' entity='mgr.smithi037.lykblh' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi037.lykblh/trash_purge_schedule"}]: dispatch 2024-09-20T22:10:47.981 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:47 smithi037 ceph-mon[23718]: [20/Sep/2024:22:10:47] ENGINE Bus STARTING 2024-09-20T22:10:47.981 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:47 smithi037 ceph-mon[23718]: [20/Sep/2024:22:10:47] ENGINE Serving on https://172.21.15.37:7150 2024-09-20T22:10:47.981 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:47 smithi037 ceph-mon[23718]: [20/Sep/2024:22:10:47] ENGINE Bus STARTED 2024-09-20T22:10:47.981 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:47 smithi037 ceph-mon[23718]: from='mgr.14162 172.21.15.37:0/1762807819' entity='mgr.smithi037.lykblh' 2024-09-20T22:10:48.595 INFO:teuthology.orchestra.run.smithi037.stderr:mgr epoch 9 is available 2024-09-20T22:10:48.595 INFO:teuthology.orchestra.run.smithi037.stderr:Generating a dashboard self-signed certificate... 2024-09-20T22:10:49.155 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:48 smithi037 ceph-mon[23718]: from='client.14166 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-09-20T22:10:49.155 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:48 smithi037 ceph-mon[23718]: mgrmap e11: smithi037.lykblh(active, since 1.05658s) 2024-09-20T22:10:49.155 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:48 smithi037 ceph-mon[23718]: from='client.14166 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-09-20T22:10:49.155 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:48 smithi037 ceph-mon[23718]: from='mgr.14162 172.21.15.37:0/1762807819' entity='mgr.smithi037.lykblh' 2024-09-20T22:10:49.315 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: Self-signed certificate created 2024-09-20T22:10:49.943 INFO:teuthology.orchestra.run.smithi037.stderr:Creating initial admin user... 2024-09-20T22:10:50.394 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:50 smithi037 ceph-mon[23718]: from='client.14174 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:10:50.394 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:50 smithi037 ceph-mon[23718]: from='mgr.14162 172.21.15.37:0/1762807819' entity='mgr.smithi037.lykblh' 2024-09-20T22:10:50.394 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:50 smithi037 ceph-mon[23718]: from='mgr.14162 172.21.15.37:0/1762807819' entity='mgr.smithi037.lykblh' 2024-09-20T22:10:50.394 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:50 smithi037 ceph-mon[23718]: mgrmap e12: smithi037.lykblh(active, since 2s) 2024-09-20T22:10:50.394 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:50 smithi037 ceph-mon[23718]: from='mgr.14162 172.21.15.37:0/1762807819' entity='mgr.smithi037.lykblh' 2024-09-20T22:10:50.881 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: {"username": "admin", "password": "$2b$12$fMVG.OF52oFKB3X1Fkh7Ieckc529AoGar13aySAbtBkGDxg6TNB2i", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1726870250, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-09-20T22:10:52.119 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:51 smithi037 ceph-mon[23718]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:10:52.120 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:51 smithi037 ceph-mon[23718]: from='mgr.14162 172.21.15.37:0/1762807819' entity='mgr.smithi037.lykblh' 2024-09-20T22:10:52.122 INFO:teuthology.orchestra.run.smithi037.stderr:Fetching dashboard port number... 2024-09-20T22:10:52.771 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: 8443 2024-09-20T22:10:53.125 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:52 smithi037 ceph-mon[23718]: from='client.? 172.21.15.37:0/1761689288' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-09-20T22:10:53.448 INFO:teuthology.orchestra.run.smithi037.stderr:systemctl: disabled 2024-09-20T22:10:53.457 INFO:teuthology.orchestra.run.smithi037.stderr:systemctl: inactive 2024-09-20T22:10:53.457 INFO:teuthology.orchestra.run.smithi037.stderr:firewalld.service is not enabled 2024-09-20T22:10:53.457 INFO:teuthology.orchestra.run.smithi037.stderr:Not possible to open ports <[8443]>. firewalld.service is not available 2024-09-20T22:10:53.460 INFO:teuthology.orchestra.run.smithi037.stderr:Ceph Dashboard is now available at: 2024-09-20T22:10:53.460 INFO:teuthology.orchestra.run.smithi037.stderr: 2024-09-20T22:10:53.460 INFO:teuthology.orchestra.run.smithi037.stderr: URL: https://smithi037.front.sepia.ceph.com:8443/ 2024-09-20T22:10:53.460 INFO:teuthology.orchestra.run.smithi037.stderr: User: admin 2024-09-20T22:10:53.460 INFO:teuthology.orchestra.run.smithi037.stderr: Password: egajrtridk 2024-09-20T22:10:53.461 INFO:teuthology.orchestra.run.smithi037.stderr: 2024-09-20T22:10:53.461 INFO:teuthology.orchestra.run.smithi037.stderr:Enabling autotune for osd_memory_target 2024-09-20T22:10:54.140 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:53 smithi037 ceph-mon[23718]: mgrmap e13: smithi037.lykblh(active, since 6s) 2024-09-20T22:10:55.550 INFO:teuthology.orchestra.run.smithi037.stderr:/usr/bin/ceph: set mgr/dashboard/cluster/status 2024-09-20T22:10:56.167 INFO:teuthology.orchestra.run.smithi037.stderr:You can access the Ceph CLI with: 2024-09-20T22:10:56.167 INFO:teuthology.orchestra.run.smithi037.stderr: 2024-09-20T22:10:56.167 INFO:teuthology.orchestra.run.smithi037.stderr: sudo /home/ubuntu/cephtest/cephadm shell --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-09-20T22:10:56.167 INFO:teuthology.orchestra.run.smithi037.stderr: 2024-09-20T22:10:56.167 INFO:teuthology.orchestra.run.smithi037.stderr:Please consider enabling telemetry to help improve Ceph: 2024-09-20T22:10:56.167 INFO:teuthology.orchestra.run.smithi037.stderr: 2024-09-20T22:10:56.167 INFO:teuthology.orchestra.run.smithi037.stderr: ceph telemetry on 2024-09-20T22:10:56.167 INFO:teuthology.orchestra.run.smithi037.stderr: 2024-09-20T22:10:56.168 INFO:teuthology.orchestra.run.smithi037.stderr:For more information see: 2024-09-20T22:10:56.168 INFO:teuthology.orchestra.run.smithi037.stderr: 2024-09-20T22:10:56.168 INFO:teuthology.orchestra.run.smithi037.stderr: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-09-20T22:10:56.168 INFO:teuthology.orchestra.run.smithi037.stderr: 2024-09-20T22:10:56.168 INFO:teuthology.orchestra.run.smithi037.stderr:Bootstrap complete. 2024-09-20T22:10:56.204 INFO:tasks.cephadm:Fetching config... 2024-09-20T22:10:56.204 DEBUG:teuthology.orchestra.run.smithi037:> set -ex 2024-09-20T22:10:56.204 DEBUG:teuthology.orchestra.run.smithi037:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-09-20T22:10:56.223 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-09-20T22:10:56.223 DEBUG:teuthology.orchestra.run.smithi037:> set -ex 2024-09-20T22:10:56.223 DEBUG:teuthology.orchestra.run.smithi037:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-09-20T22:10:56.280 INFO:tasks.cephadm:Fetching mon keyring... 2024-09-20T22:10:56.280 DEBUG:teuthology.orchestra.run.smithi037:> set -ex 2024-09-20T22:10:56.280 DEBUG:teuthology.orchestra.run.smithi037:> sudo dd if=/var/lib/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/mon.smithi037/keyring of=/dev/stdout 2024-09-20T22:10:56.346 INFO:tasks.cephadm:Fetching pub ssh key... 2024-09-20T22:10:56.346 DEBUG:teuthology.orchestra.run.smithi037:> set -ex 2024-09-20T22:10:56.346 DEBUG:teuthology.orchestra.run.smithi037:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-09-20T22:10:56.402 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-09-20T22:10:56.402 DEBUG:teuthology.orchestra.run.smithi037:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCvwq6Bv1SrRrqqnGsaspxqsm7ELTL2SSveARgkraAk5Om/8TMKeQKiW4PdHopgBggVe2e0pPbmNRc7lCD/manhAsEz8pD79ITydPGuBd1YzAE1WU80cxNO5Jgh7I+/FqQK9alwJxoEqEg1FRInU+Jy7vK1NJbN8+lZYSSbA7WeuBnV277TMGTByrOioFYHbCCPHClaCCVfO/af9HpWJW+JOmrpm5PHOAlTaZyFXFOvbGMyjOs+E2dNJgvtjpQOcwxrYrpln/Dk2KO6/ICXlxaQr57N2DNzQG+KXrl7B+1IKHgOgCez9wG9vKLR+CYxiniaxQcP5M1i/lFOCvxBEYg2GOqdmYJ7dPzRGasSvv92+mFkWMewmHsriieo/IH+9akLgcjy+DX/8gfxASzb8D8+B8vU7IrKDYIBFo7Eo0IWpRU0dQTwGBSYo8uHjoluGWRGrwgTlbSEzbKy4gGiTLXjNYlaVqdKaN9A30wJdQbCimaLIT9ZBIvP07hMYUhpDGc= ceph-f9a6357a-779c-11ef-baf5-efdc52797490' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-09-20T22:10:56.481 INFO:teuthology.orchestra.run.smithi037.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCvwq6Bv1SrRrqqnGsaspxqsm7ELTL2SSveARgkraAk5Om/8TMKeQKiW4PdHopgBggVe2e0pPbmNRc7lCD/manhAsEz8pD79ITydPGuBd1YzAE1WU80cxNO5Jgh7I+/FqQK9alwJxoEqEg1FRInU+Jy7vK1NJbN8+lZYSSbA7WeuBnV277TMGTByrOioFYHbCCPHClaCCVfO/af9HpWJW+JOmrpm5PHOAlTaZyFXFOvbGMyjOs+E2dNJgvtjpQOcwxrYrpln/Dk2KO6/ICXlxaQr57N2DNzQG+KXrl7B+1IKHgOgCez9wG9vKLR+CYxiniaxQcP5M1i/lFOCvxBEYg2GOqdmYJ7dPzRGasSvv92+mFkWMewmHsriieo/IH+9akLgcjy+DX/8gfxASzb8D8+B8vU7IrKDYIBFo7Eo0IWpRU0dQTwGBSYo8uHjoluGWRGrwgTlbSEzbKy4gGiTLXjNYlaVqdKaN9A30wJdQbCimaLIT9ZBIvP07hMYUhpDGc= ceph-f9a6357a-779c-11ef-baf5-efdc52797490 2024-09-20T22:10:56.493 DEBUG:teuthology.orchestra.run.smithi194:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCvwq6Bv1SrRrqqnGsaspxqsm7ELTL2SSveARgkraAk5Om/8TMKeQKiW4PdHopgBggVe2e0pPbmNRc7lCD/manhAsEz8pD79ITydPGuBd1YzAE1WU80cxNO5Jgh7I+/FqQK9alwJxoEqEg1FRInU+Jy7vK1NJbN8+lZYSSbA7WeuBnV277TMGTByrOioFYHbCCPHClaCCVfO/af9HpWJW+JOmrpm5PHOAlTaZyFXFOvbGMyjOs+E2dNJgvtjpQOcwxrYrpln/Dk2KO6/ICXlxaQr57N2DNzQG+KXrl7B+1IKHgOgCez9wG9vKLR+CYxiniaxQcP5M1i/lFOCvxBEYg2GOqdmYJ7dPzRGasSvv92+mFkWMewmHsriieo/IH+9akLgcjy+DX/8gfxASzb8D8+B8vU7IrKDYIBFo7Eo0IWpRU0dQTwGBSYo8uHjoluGWRGrwgTlbSEzbKy4gGiTLXjNYlaVqdKaN9A30wJdQbCimaLIT9ZBIvP07hMYUhpDGc= ceph-f9a6357a-779c-11ef-baf5-efdc52797490' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-09-20T22:10:56.600 INFO:teuthology.orchestra.run.smithi194.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCvwq6Bv1SrRrqqnGsaspxqsm7ELTL2SSveARgkraAk5Om/8TMKeQKiW4PdHopgBggVe2e0pPbmNRc7lCD/manhAsEz8pD79ITydPGuBd1YzAE1WU80cxNO5Jgh7I+/FqQK9alwJxoEqEg1FRInU+Jy7vK1NJbN8+lZYSSbA7WeuBnV277TMGTByrOioFYHbCCPHClaCCVfO/af9HpWJW+JOmrpm5PHOAlTaZyFXFOvbGMyjOs+E2dNJgvtjpQOcwxrYrpln/Dk2KO6/ICXlxaQr57N2DNzQG+KXrl7B+1IKHgOgCez9wG9vKLR+CYxiniaxQcP5M1i/lFOCvxBEYg2GOqdmYJ7dPzRGasSvv92+mFkWMewmHsriieo/IH+9akLgcjy+DX/8gfxASzb8D8+B8vU7IrKDYIBFo7Eo0IWpRU0dQTwGBSYo8uHjoluGWRGrwgTlbSEzbKy4gGiTLXjNYlaVqdKaN9A30wJdQbCimaLIT9ZBIvP07hMYUhpDGc= ceph-f9a6357a-779c-11ef-baf5-efdc52797490 2024-09-20T22:10:56.613 DEBUG:teuthology.orchestra.run.smithi037:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-09-20T22:10:56.718 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:10:56 smithi037 ceph-mon[23718]: from='client.? 172.21.15.37:0/2870333149' entity='client.admin' 2024-09-20T22:11:00.870 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-09-20T22:11:00.871 DEBUG:teuthology.orchestra.run.smithi037:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-09-20T22:11:01.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:00 smithi037 ceph-mon[23718]: from='client.? 172.21.15.37:0/2093028829' entity='client.admin' 2024-09-20T22:11:01.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:00 smithi037 ceph-mon[23718]: from='mgr.14162 172.21.15.37:0/1762807819' entity='mgr.smithi037.lykblh' 2024-09-20T22:11:01.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:00 smithi037 ceph-mon[23718]: from='mgr.14162 172.21.15.37:0/1762807819' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "who": "osd/host:smithi037", "name": "osd_memory_target"}]: dispatch 2024-09-20T22:11:01.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:00 smithi037 ceph-mon[23718]: from='mgr.14162 172.21.15.37:0/1762807819' entity='mgr.smithi037.lykblh' 2024-09-20T22:11:01.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:00 smithi037 ceph-mon[23718]: Deploying daemon alertmanager.smithi037 on smithi037 2024-09-20T22:11:04.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:04 smithi037 ceph-mon[23718]: from='client.14186 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:11:04.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:04 smithi037 ceph-mon[23718]: from='mgr.14162 172.21.15.37:0/1762807819' entity='mgr.smithi037.lykblh' 2024-09-20T22:11:04.961 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi194 2024-09-20T22:11:04.961 DEBUG:teuthology.orchestra.run.smithi194:> set -ex 2024-09-20T22:11:04.962 DEBUG:teuthology.orchestra.run.smithi194:> dd of=/etc/ceph/ceph.conf 2024-09-20T22:11:04.981 DEBUG:teuthology.orchestra.run.smithi194:> set -ex 2024-09-20T22:11:04.981 DEBUG:teuthology.orchestra.run.smithi194:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-20T22:11:05.040 INFO:tasks.cephadm:Adding host smithi194 to orchestrator... 2024-09-20T22:11:05.040 DEBUG:teuthology.orchestra.run.smithi037:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -- ceph orch host add smithi194 2024-09-20T22:11:08.108 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:07 smithi037 ceph-mon[23718]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T22:11:09.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:08 smithi037 ceph-mon[23718]: from='client.14188 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi194", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:11:09.306 INFO:teuthology.orchestra.run.smithi037.stdout:Added host 'smithi194' with addr '172.21.15.194' 2024-09-20T22:11:09.977 DEBUG:teuthology.orchestra.run.smithi037:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -- ceph orch host ls --format=json 2024-09-20T22:11:10.523 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:10 smithi037 ceph-mon[23718]: Deploying cephadm binary to smithi194 2024-09-20T22:11:10.523 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:10 smithi037 ceph-mon[23718]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T22:11:10.523 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:10 smithi037 ceph-mon[23718]: from='mgr.14162 172.21.15.37:0/1762807819' entity='mgr.smithi037.lykblh' 2024-09-20T22:11:10.523 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:10 smithi037 ceph-mon[23718]: Added host smithi194 2024-09-20T22:11:12.075 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:11 smithi037 ceph-mon[23718]: from='mgr.14162 172.21.15.37:0/1762807819' entity='mgr.smithi037.lykblh' 2024-09-20T22:11:12.075 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:11 smithi037 ceph-mon[23718]: from='mgr.14162 172.21.15.37:0/1762807819' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi037", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-20T22:11:12.075 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:11 smithi037 ceph-mon[23718]: from='mgr.14162 172.21.15.37:0/1762807819' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi037", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-09-20T22:11:12.075 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:11 smithi037 ceph-mon[23718]: from='mgr.14162 172.21.15.37:0/1762807819' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:11:12.075 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:11 smithi037 ceph-mon[23718]: Deploying daemon crash.smithi037 on smithi037 2024-09-20T22:11:12.075 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:11 smithi037 ceph-mon[23718]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T22:11:12.506 INFO:teuthology.orchestra.run.smithi037.stdout: 2024-09-20T22:11:12.506 INFO:teuthology.orchestra.run.smithi037.stdout:[{"addr": "172.21.15.37", "hostname": "smithi037", "labels": [], "status": ""}, {"addr": "172.21.15.194", "hostname": "smithi194", "labels": [], "status": ""}] 2024-09-20T22:11:13.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:12 smithi037 ceph-mon[23718]: from='mgr.14162 172.21.15.37:0/1762807819' entity='mgr.smithi037.lykblh' 2024-09-20T22:11:13.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:12 smithi037 ceph-mon[23718]: from='client.14190 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-20T22:11:13.619 INFO:tasks.cephadm:Setting crush tunables to default 2024-09-20T22:11:13.620 DEBUG:teuthology.orchestra.run.smithi037:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -- ceph osd crush tunables default 2024-09-20T22:11:14.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:13 smithi037 ceph-mon[23718]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T22:11:16.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:15 smithi037 ceph-mon[23718]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T22:11:16.905 INFO:teuthology.orchestra.run.smithi037.stderr:adjusted tunables profile to default 2024-09-20T22:11:17.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:16 smithi037 ceph-mon[23718]: from='client.? 172.21.15.37:0/1461079359' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-09-20T22:11:17.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:16 smithi037 ceph-mon[23718]: from='mgr.14162 172.21.15.37:0/1762807819' entity='mgr.smithi037.lykblh' 2024-09-20T22:11:17.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:16 smithi037 ceph-mon[23718]: from='mgr.14162 172.21.15.37:0/1762807819' entity='mgr.smithi037.lykblh' 2024-09-20T22:11:17.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:16 smithi037 ceph-mon[23718]: from='mgr.14162 172.21.15.37:0/1762807819' entity='mgr.smithi037.lykblh' 2024-09-20T22:11:17.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:16 smithi037 ceph-mon[23718]: from='mgr.14162 172.21.15.37:0/1762807819' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-20T22:11:17.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:16 smithi037 ceph-mon[23718]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-20T22:11:17.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:16 smithi037 ceph-mon[23718]: from='mgr.14162 172.21.15.37:0/1762807819' entity='mgr.smithi037.lykblh' 2024-09-20T22:11:17.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:16 smithi037 ceph-mon[23718]: Deploying daemon grafana.smithi037 on smithi037 2024-09-20T22:11:17.577 INFO:tasks.cephadm:Adding mon.smithi037 on smithi037 2024-09-20T22:11:17.577 INFO:tasks.cephadm:Adding mon.smithi194 on smithi194 2024-09-20T22:11:17.578 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -- ceph orch apply mon '2;smithi037:172.21.15.37=smithi037;smithi194:172.21.15.194=smithi194' 2024-09-20T22:11:18.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:17 smithi037 ceph-mon[23718]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T22:11:18.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:17 smithi037 ceph-mon[23718]: from='mgr.14162 172.21.15.37:0/1762807819' entity='mgr.smithi037.lykblh' 2024-09-20T22:11:18.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:17 smithi037 ceph-mon[23718]: from='client.? 172.21.15.37:0/1461079359' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-09-20T22:11:18.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:17 smithi037 ceph-mon[23718]: osdmap e4: 0 total, 0 up, 0 in 2024-09-20T22:11:19.392 INFO:teuthology.orchestra.run.smithi194.stdout:Scheduled mon update... 2024-09-20T22:11:19.968 DEBUG:teuthology.orchestra.run.smithi194:mon.smithi194> sudo journalctl -f -n 0 -u ceph-f9a6357a-779c-11ef-baf5-efdc52797490@mon.smithi194.service 2024-09-20T22:11:19.970 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-20T22:11:19.971 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -- ceph mon dump -f json 2024-09-20T22:11:20.662 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:20 smithi037 ceph-mon[23718]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T22:11:20.662 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:20 smithi037 ceph-mon[23718]: from='client.14194 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "2;smithi037:172.21.15.37=smithi037;smithi194:172.21.15.194=smithi194", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:11:20.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:20 smithi037 ceph-mon[23718]: Saving service mon spec with placement smithi037:172.21.15.37=smithi037;smithi194:172.21.15.194=smithi194;count:2 2024-09-20T22:11:20.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:20 smithi037 ceph-mon[23718]: from='mgr.14162 172.21.15.37:0/1762807819' entity='mgr.smithi037.lykblh' 2024-09-20T22:11:21.644 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-09-20T22:11:21.645 INFO:teuthology.orchestra.run.smithi194.stdout:{"epoch":1,"fsid":"f9a6357a-779c-11ef-baf5-efdc52797490","modified":"2024-09-20T22:09:36.885927Z","created":"2024-09-20T22:09:36.885927Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi037","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:3300","nonce":0},{"type":"v1","addr":"172.21.15.37:6789","nonce":0}]},"addr":"172.21.15.37:6789/0","public_addr":"172.21.15.37:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-20T22:11:21.646 INFO:teuthology.orchestra.run.smithi194.stderr:dumped monmap epoch 1 2024-09-20T22:11:22.662 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:22 smithi037 ceph-mon[23718]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T22:11:22.662 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:22 smithi037 ceph-mon[23718]: from='client.? 172.21.15.194:0/3203044076' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-20T22:11:23.218 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-20T22:11:23.218 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -- ceph mon dump -f json 2024-09-20T22:11:24.662 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:24 smithi037 ceph-mon[23718]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T22:11:25.074 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-09-20T22:11:25.074 INFO:teuthology.orchestra.run.smithi194.stdout:{"epoch":1,"fsid":"f9a6357a-779c-11ef-baf5-efdc52797490","modified":"2024-09-20T22:09:36.885927Z","created":"2024-09-20T22:09:36.885927Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi037","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:3300","nonce":0},{"type":"v1","addr":"172.21.15.37:6789","nonce":0}]},"addr":"172.21.15.37:6789/0","public_addr":"172.21.15.37:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-20T22:11:25.076 INFO:teuthology.orchestra.run.smithi194.stderr:dumped monmap epoch 1 2024-09-20T22:11:25.634 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:25 smithi037 ceph-mon[23718]: from='client.? 172.21.15.194:0/2180289217' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-20T22:11:26.662 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:26 smithi037 ceph-mon[23718]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T22:11:26.675 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-20T22:11:26.675 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -- ceph mon dump -f json 2024-09-20T22:11:28.306 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-09-20T22:11:28.306 INFO:teuthology.orchestra.run.smithi194.stdout:{"epoch":1,"fsid":"f9a6357a-779c-11ef-baf5-efdc52797490","modified":"2024-09-20T22:09:36.885927Z","created":"2024-09-20T22:09:36.885927Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi037","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:3300","nonce":0},{"type":"v1","addr":"172.21.15.37:6789","nonce":0}]},"addr":"172.21.15.37:6789/0","public_addr":"172.21.15.37:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-20T22:11:28.308 INFO:teuthology.orchestra.run.smithi194.stderr:dumped monmap epoch 1 2024-09-20T22:11:28.412 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:28 smithi037 ceph-mon[23718]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T22:11:28.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:28 smithi037 ceph-mon[23718]: from='mgr.14162 172.21.15.37:0/1762807819' entity='mgr.smithi037.lykblh' 2024-09-20T22:11:28.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:28 smithi037 ceph-mon[23718]: from='client.? 172.21.15.194:0/3399831468' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-20T22:11:29.662 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:29 smithi037 ceph-mon[23718]: Deploying daemon node-exporter.smithi037 on smithi037 2024-09-20T22:11:29.893 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-20T22:11:29.893 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -- ceph mon dump -f json 2024-09-20T22:11:30.495 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:30 smithi037 ceph-mon[23718]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T22:11:31.666 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-09-20T22:11:31.667 INFO:teuthology.orchestra.run.smithi194.stdout:{"epoch":1,"fsid":"f9a6357a-779c-11ef-baf5-efdc52797490","modified":"2024-09-20T22:09:36.885927Z","created":"2024-09-20T22:09:36.885927Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi037","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:3300","nonce":0},{"type":"v1","addr":"172.21.15.37:6789","nonce":0}]},"addr":"172.21.15.37:6789/0","public_addr":"172.21.15.37:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-20T22:11:31.668 INFO:teuthology.orchestra.run.smithi194.stderr:dumped monmap epoch 1 2024-09-20T22:11:32.000 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:31 smithi037 ceph-mon[23718]: from='mgr.14162 172.21.15.37:0/1762807819' entity='mgr.smithi037.lykblh' 2024-09-20T22:11:32.000 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:31 smithi037 ceph-mon[23718]: from='mgr.14162 172.21.15.37:0/1762807819' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-20T22:11:32.000 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:31 smithi037 ceph-mon[23718]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T22:11:32.000 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:31 smithi037 ceph-mon[23718]: from='client.? 172.21.15.194:0/126376549' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-20T22:11:32.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:32 smithi037 ceph-mon[23718]: from='mgr.14162 172.21.15.37:0/1762807819' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-20T22:11:32.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:32 smithi037 ceph-mon[23718]: mgrmap e14: smithi037.lykblh(active, since 44s) 2024-09-20T22:11:33.221 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-20T22:11:33.222 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -- ceph mon dump -f json 2024-09-20T22:11:35.067 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-09-20T22:11:35.068 INFO:teuthology.orchestra.run.smithi194.stdout:{"epoch":1,"fsid":"f9a6357a-779c-11ef-baf5-efdc52797490","modified":"2024-09-20T22:09:36.885927Z","created":"2024-09-20T22:09:36.885927Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi037","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:3300","nonce":0},{"type":"v1","addr":"172.21.15.37:6789","nonce":0}]},"addr":"172.21.15.37:6789/0","public_addr":"172.21.15.37:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-20T22:11:35.069 INFO:teuthology.orchestra.run.smithi194.stderr:dumped monmap epoch 1 2024-09-20T22:11:35.412 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:35 smithi037 ceph-mon[23718]: from='client.? 172.21.15.194:0/3832956007' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-20T22:11:36.606 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-20T22:11:36.607 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -- ceph mon dump -f json 2024-09-20T22:11:38.325 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-09-20T22:11:38.325 INFO:teuthology.orchestra.run.smithi194.stdout:{"epoch":1,"fsid":"f9a6357a-779c-11ef-baf5-efdc52797490","modified":"2024-09-20T22:09:36.885927Z","created":"2024-09-20T22:09:36.885927Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi037","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:3300","nonce":0},{"type":"v1","addr":"172.21.15.37:6789","nonce":0}]},"addr":"172.21.15.37:6789/0","public_addr":"172.21.15.37:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-20T22:11:38.326 INFO:teuthology.orchestra.run.smithi194.stderr:dumped monmap epoch 1 2024-09-20T22:11:39.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:39 smithi037 ceph-mon[23718]: Active manager daemon smithi037.lykblh restarted 2024-09-20T22:11:39.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:39 smithi037 ceph-mon[23718]: Activating manager daemon smithi037.lykblh 2024-09-20T22:11:39.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:39 smithi037 ceph-mon[23718]: osdmap e5: 0 total, 0 up, 0 in 2024-09-20T22:11:39.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:39 smithi037 ceph-mon[23718]: mgrmap e15: smithi037.lykblh(active, starting, since 0.0550607s) 2024-09-20T22:11:39.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:39 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mon metadata", "id": "smithi037"}]: dispatch 2024-09-20T22:11:39.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:39 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mgr metadata", "who": "smithi037.lykblh", "id": "smithi037.lykblh"}]: dispatch 2024-09-20T22:11:39.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:39 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-20T22:11:39.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:39 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-20T22:11:39.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:39 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-20T22:11:39.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:39 smithi037 ceph-mon[23718]: Manager daemon smithi037.lykblh is now available 2024-09-20T22:11:39.414 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:39 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:11:39.414 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:39 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:11:39.414 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:39 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:11:39.414 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:39 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:11:39.414 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:39 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:11:39.414 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:39 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi037.lykblh/mirror_snapshot_schedule"}]: dispatch 2024-09-20T22:11:39.414 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:39 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi037.lykblh/trash_purge_schedule"}]: dispatch 2024-09-20T22:11:39.414 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:39 smithi037 ceph-mon[23718]: from='client.? 172.21.15.194:0/748363259' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-20T22:11:39.414 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:39 smithi037 ceph-mon[23718]: [20/Sep/2024:22:11:38] ENGINE Bus STARTING 2024-09-20T22:11:39.414 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:39 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:11:40.000 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-20T22:11:40.001 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -- ceph mon dump -f json 2024-09-20T22:11:40.372 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:40 smithi037 ceph-mon[23718]: [20/Sep/2024:22:11:38] ENGINE Serving on https://172.21.15.37:7150 2024-09-20T22:11:40.372 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:40 smithi037 ceph-mon[23718]: [20/Sep/2024:22:11:38] ENGINE Bus STARTED 2024-09-20T22:11:40.372 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:40 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:11:40.372 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:40 smithi037 ceph-mon[23718]: mgrmap e16: smithi037.lykblh(active, since 1.05794s) 2024-09-20T22:11:40.373 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:40 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:11:41.412 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:41 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:11:41.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:41 smithi037 ceph-mon[23718]: mgrmap e17: smithi037.lykblh(active, since 2s) 2024-09-20T22:11:41.722 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-09-20T22:11:41.723 INFO:teuthology.orchestra.run.smithi194.stdout:{"epoch":1,"fsid":"f9a6357a-779c-11ef-baf5-efdc52797490","modified":"2024-09-20T22:09:36.885927Z","created":"2024-09-20T22:09:36.885927Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi037","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:3300","nonce":0},{"type":"v1","addr":"172.21.15.37:6789","nonce":0}]},"addr":"172.21.15.37:6789/0","public_addr":"172.21.15.37:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-20T22:11:41.723 INFO:teuthology.orchestra.run.smithi194.stderr:dumped monmap epoch 1 2024-09-20T22:11:42.662 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:42 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:11:42.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:42 smithi037 ceph-mon[23718]: from='client.? 172.21.15.194:0/2843179036' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-20T22:11:42.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:42 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:11:42.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:42 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "who": "osd/host:smithi037", "name": "osd_memory_target"}]: dispatch 2024-09-20T22:11:42.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:42 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:11:43.387 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-20T22:11:43.387 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -- ceph mon dump -f json 2024-09-20T22:11:43.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:43 smithi037 ceph-mon[23718]: Updating smithi037:/etc/ceph/ceph.conf 2024-09-20T22:11:43.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:43 smithi037 ceph-mon[23718]: Updating smithi037:/etc/ceph/ceph.client.admin.keyring 2024-09-20T22:11:45.445 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-09-20T22:11:45.445 INFO:teuthology.orchestra.run.smithi194.stdout:{"epoch":1,"fsid":"f9a6357a-779c-11ef-baf5-efdc52797490","modified":"2024-09-20T22:09:36.885927Z","created":"2024-09-20T22:09:36.885927Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi037","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:3300","nonce":0},{"type":"v1","addr":"172.21.15.37:6789","nonce":0}]},"addr":"172.21.15.37:6789/0","public_addr":"172.21.15.37:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-20T22:11:45.447 INFO:teuthology.orchestra.run.smithi194.stderr:dumped monmap epoch 1 2024-09-20T22:11:45.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:45 smithi037 ceph-mon[23718]: from='client.? 172.21.15.194:0/409643448' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-20T22:11:46.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:46 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:11:46.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:46 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:11:46.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:46 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "who": "osd/host:smithi194", "name": "osd_memory_target"}]: dispatch 2024-09-20T22:11:46.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:46 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:11:46.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:46 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi194", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-20T22:11:46.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:46 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi194", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-09-20T22:11:46.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:46 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:11:46.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:46 smithi037 ceph-mon[23718]: Deploying daemon crash.smithi194 on smithi194 2024-09-20T22:11:47.086 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-20T22:11:47.086 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -- ceph mon dump -f json 2024-09-20T22:11:49.178 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-09-20T22:11:49.178 INFO:teuthology.orchestra.run.smithi194.stdout:{"epoch":1,"fsid":"f9a6357a-779c-11ef-baf5-efdc52797490","modified":"2024-09-20T22:09:36.885927Z","created":"2024-09-20T22:09:36.885927Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi037","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:3300","nonce":0},{"type":"v1","addr":"172.21.15.37:6789","nonce":0}]},"addr":"172.21.15.37:6789/0","public_addr":"172.21.15.37:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-20T22:11:49.180 INFO:teuthology.orchestra.run.smithi194.stderr:dumped monmap epoch 1 2024-09-20T22:11:49.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:49 smithi037 ceph-mon[23718]: from='client.? 172.21.15.194:0/2766039605' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-20T22:11:49.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:49 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:11:49.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:49 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi194.bkcglt", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-20T22:11:49.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:49 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.smithi194.bkcglt", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-20T22:11:49.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:49 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-20T22:11:49.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:49 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:11:50.899 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-20T22:11:50.900 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -- ceph mon dump -f json 2024-09-20T22:11:50.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:50 smithi037 ceph-mon[23718]: Deploying daemon mgr.smithi194.bkcglt on smithi194 2024-09-20T22:11:53.303 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-09-20T22:11:53.304 INFO:teuthology.orchestra.run.smithi194.stdout:{"epoch":1,"fsid":"f9a6357a-779c-11ef-baf5-efdc52797490","modified":"2024-09-20T22:09:36.885927Z","created":"2024-09-20T22:09:36.885927Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi037","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:3300","nonce":0},{"type":"v1","addr":"172.21.15.37:6789","nonce":0}]},"addr":"172.21.15.37:6789/0","public_addr":"172.21.15.37:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-20T22:11:53.306 INFO:teuthology.orchestra.run.smithi194.stderr:dumped monmap epoch 1 2024-09-20T22:11:54.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:53 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:11:54.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:53 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-20T22:11:54.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:53 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:11:54.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:53 smithi037 ceph-mon[23718]: Deploying daemon mon.smithi194 on smithi194 2024-09-20T22:11:54.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:53 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:11:54.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:53 smithi037 ceph-mon[23718]: from='client.? 172.21.15.194:0/1569298410' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-20T22:11:55.856 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-20T22:11:55.857 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -- ceph mon dump -f json 2024-09-20T22:11:57.768 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-09-20T22:11:57.768 INFO:teuthology.orchestra.run.smithi194.stdout:{"epoch":1,"fsid":"f9a6357a-779c-11ef-baf5-efdc52797490","modified":"2024-09-20T22:09:36.885927Z","created":"2024-09-20T22:09:36.885927Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi037","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:3300","nonce":0},{"type":"v1","addr":"172.21.15.37:6789","nonce":0}]},"addr":"172.21.15.37:6789/0","public_addr":"172.21.15.37:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-20T22:11:57.770 INFO:teuthology.orchestra.run.smithi194.stderr:dumped monmap epoch 1 2024-09-20T22:11:58.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:11:57 smithi037 ceph-mon[23718]: from='client.? 172.21.15.194:0/2462019382' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-20T22:11:59.410 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-20T22:11:59.411 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -- ceph mon dump -f json 2024-09-20T22:12:03.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:03 smithi037 ceph-mon[23718]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T22:12:03.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:03 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mon metadata", "id": "smithi037"}]: dispatch 2024-09-20T22:12:03.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:03 smithi037 ceph-mon[23718]: mon.smithi037 calling monitor election 2024-09-20T22:12:03.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:03 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mon metadata", "id": "smithi194"}]: dispatch 2024-09-20T22:12:03.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:03 smithi037 ceph-mon[23718]: from='mgr.? 172.21.15.194:0/2548712185' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi194.bkcglt/crt"}]: dispatch 2024-09-20T22:12:03.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:03 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mon metadata", "id": "smithi194"}]: dispatch 2024-09-20T22:12:03.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:03 smithi037 ceph-mon[23718]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T22:12:03.914 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:03 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mon metadata", "id": "smithi194"}]: dispatch 2024-09-20T22:12:03.914 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:03 smithi037 ceph-mon[23718]: mon.smithi194 calling monitor election 2024-09-20T22:12:03.914 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:03 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mon metadata", "id": "smithi194"}]: dispatch 2024-09-20T22:12:03.914 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:03 smithi037 ceph-mon[23718]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T22:12:03.914 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:03 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mon metadata", "id": "smithi194"}]: dispatch 2024-09-20T22:12:03.914 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:03 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mon metadata", "id": "smithi194"}]: dispatch 2024-09-20T22:12:03.914 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:03 smithi037 ceph-mon[23718]: mon.smithi037 is new leader, mons smithi037,smithi194 in quorum (ranks 0,1) 2024-09-20T22:12:03.914 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:03 smithi037 ceph-mon[23718]: monmap e2: 2 mons at {smithi037=[v2:172.21.15.37:3300/0,v1:172.21.15.37:6789/0],smithi194=[v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0]} 2024-09-20T22:12:03.914 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:03 smithi037 ceph-mon[23718]: fsmap 2024-09-20T22:12:03.914 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:03 smithi037 ceph-mon[23718]: osdmap e5: 0 total, 0 up, 0 in 2024-09-20T22:12:03.914 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:03 smithi037 ceph-mon[23718]: mgrmap e17: smithi037.lykblh(active, since 25s) 2024-09-20T22:12:03.914 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:03 smithi037 ceph-mon[23718]: Standby manager daemon smithi194.bkcglt started 2024-09-20T22:12:03.915 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:03 smithi037 ceph-mon[23718]: from='mgr.? 172.21.15.194:0/2548712185' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-20T22:12:03.915 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:03 smithi037 ceph-mon[23718]: overall HEALTH_OK 2024-09-20T22:12:03.915 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:03 smithi037 ceph-mon[23718]: from='mgr.? 172.21.15.194:0/2548712185' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi194.bkcglt/key"}]: dispatch 2024-09-20T22:12:03.915 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:03 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:12:03.915 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:03 smithi037 ceph-mon[23718]: from='mgr.? 172.21.15.194:0/2548712185' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-20T22:12:04.335 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-09-20T22:12:04.336 INFO:teuthology.orchestra.run.smithi194.stdout:{"epoch":2,"fsid":"f9a6357a-779c-11ef-baf5-efdc52797490","modified":"2024-09-20T22:11:58.559736Z","created":"2024-09-20T22:09:36.885927Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi037","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:3300","nonce":0},{"type":"v1","addr":"172.21.15.37:6789","nonce":0}]},"addr":"172.21.15.37:6789/0","public_addr":"172.21.15.37:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"smithi194","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:3300","nonce":0},{"type":"v1","addr":"172.21.15.194:6789","nonce":0}]},"addr":"172.21.15.194:6789/0","public_addr":"172.21.15.194:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-09-20T22:12:04.337 INFO:teuthology.orchestra.run.smithi194.stderr:dumped monmap epoch 2 2024-09-20T22:12:04.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:04 smithi037 ceph-mon[23718]: Deploying daemon node-exporter.smithi194 on smithi194 2024-09-20T22:12:04.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:04 smithi037 ceph-mon[23718]: mgrmap e18: smithi037.lykblh(active, since 25s), standbys: smithi194.bkcglt 2024-09-20T22:12:04.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:04 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mgr metadata", "who": "smithi194.bkcglt", "id": "smithi194.bkcglt"}]: dispatch 2024-09-20T22:12:04.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:04 smithi037 ceph-mon[23718]: from='client.? 172.21.15.194:0/3177950260' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-20T22:12:04.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:04 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mon metadata", "id": "smithi194"}]: dispatch 2024-09-20T22:12:05.007 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-09-20T22:12:05.008 DEBUG:teuthology.orchestra.run.smithi037:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -- ceph config generate-minimal-conf 2024-09-20T22:12:05.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:05 smithi037 ceph-mon[23718]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T22:12:06.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:05 smithi194 ceph-mon[25957]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T22:12:06.945 INFO:teuthology.orchestra.run.smithi037.stdout:# minimal ceph.conf for f9a6357a-779c-11ef-baf5-efdc52797490 2024-09-20T22:12:06.945 INFO:teuthology.orchestra.run.smithi037.stdout:[global] 2024-09-20T22:12:06.945 INFO:teuthology.orchestra.run.smithi037.stdout: fsid = f9a6357a-779c-11ef-baf5-efdc52797490 2024-09-20T22:12:06.945 INFO:teuthology.orchestra.run.smithi037.stdout: mon_host = [v2:172.21.15.37:3300/0,v1:172.21.15.37:6789/0] [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] 2024-09-20T22:12:07.551 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-09-20T22:12:07.551 DEBUG:teuthology.orchestra.run.smithi037:> set -ex 2024-09-20T22:12:07.551 DEBUG:teuthology.orchestra.run.smithi037:> sudo dd of=/etc/ceph/ceph.conf 2024-09-20T22:12:07.579 DEBUG:teuthology.orchestra.run.smithi037:> set -ex 2024-09-20T22:12:07.579 DEBUG:teuthology.orchestra.run.smithi037:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-20T22:12:07.647 DEBUG:teuthology.orchestra.run.smithi194:> set -ex 2024-09-20T22:12:07.648 DEBUG:teuthology.orchestra.run.smithi194:> sudo dd of=/etc/ceph/ceph.conf 2024-09-20T22:12:07.677 DEBUG:teuthology.orchestra.run.smithi194:> set -ex 2024-09-20T22:12:07.677 DEBUG:teuthology.orchestra.run.smithi194:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-20T22:12:07.743 INFO:tasks.cephadm:Deploying OSDs... 2024-09-20T22:12:07.743 DEBUG:teuthology.orchestra.run.smithi037:> set -ex 2024-09-20T22:12:07.743 DEBUG:teuthology.orchestra.run.smithi037:> dd if=/scratch_devs of=/dev/stdout 2024-09-20T22:12:07.760 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-20T22:12:07.760 DEBUG:teuthology.orchestra.run.smithi037:> stat /dev/vg_nvme/lv_1 2024-09-20T22:12:07.816 INFO:teuthology.orchestra.run.smithi037.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-20T22:12:07.816 INFO:teuthology.orchestra.run.smithi037.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-20T22:12:07.816 INFO:teuthology.orchestra.run.smithi037.stdout:Device: 5h/5d Inode: 668 Links: 1 2024-09-20T22:12:07.816 INFO:teuthology.orchestra.run.smithi037.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-20T22:12:07.816 INFO:teuthology.orchestra.run.smithi037.stdout:Context: system_u:object_r:device_t:s0 2024-09-20T22:12:07.816 INFO:teuthology.orchestra.run.smithi037.stdout:Access: 2024-09-20 22:11:28.494937403 +0000 2024-09-20T22:12:07.816 INFO:teuthology.orchestra.run.smithi037.stdout:Modify: 2024-09-20 22:08:56.112793633 +0000 2024-09-20T22:12:07.816 INFO:teuthology.orchestra.run.smithi037.stdout:Change: 2024-09-20 22:08:56.112793633 +0000 2024-09-20T22:12:07.816 INFO:teuthology.orchestra.run.smithi037.stdout: Birth: 2024-09-20 22:08:56.112793633 +0000 2024-09-20T22:12:07.817 DEBUG:teuthology.orchestra.run.smithi037:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-20T22:12:07.877 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:07 smithi037 ceph-mon[23718]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T22:12:07.878 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:07 smithi037 ceph-mon[23718]: from='client.? 172.21.15.37:0/2598899264' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:12:07.878 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:07 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:12:07.880 INFO:teuthology.orchestra.run.smithi037.stderr:1+0 records in 2024-09-20T22:12:07.880 INFO:teuthology.orchestra.run.smithi037.stderr:1+0 records out 2024-09-20T22:12:07.880 INFO:teuthology.orchestra.run.smithi037.stderr:512 bytes copied, 0.0002052 s, 2.5 MB/s 2024-09-20T22:12:07.881 DEBUG:teuthology.orchestra.run.smithi037:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-20T22:12:07.938 DEBUG:teuthology.orchestra.run.smithi037:> stat /dev/vg_nvme/lv_2 2024-09-20T22:12:07.994 INFO:teuthology.orchestra.run.smithi037.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-20T22:12:07.994 INFO:teuthology.orchestra.run.smithi037.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-20T22:12:07.994 INFO:teuthology.orchestra.run.smithi037.stdout:Device: 5h/5d Inode: 687 Links: 1 2024-09-20T22:12:07.994 INFO:teuthology.orchestra.run.smithi037.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-20T22:12:07.994 INFO:teuthology.orchestra.run.smithi037.stdout:Context: system_u:object_r:device_t:s0 2024-09-20T22:12:07.994 INFO:teuthology.orchestra.run.smithi037.stdout:Access: 2024-09-20 22:11:28.494937403 +0000 2024-09-20T22:12:07.994 INFO:teuthology.orchestra.run.smithi037.stdout:Modify: 2024-09-20 22:08:56.455788581 +0000 2024-09-20T22:12:07.994 INFO:teuthology.orchestra.run.smithi037.stdout:Change: 2024-09-20 22:08:56.455788581 +0000 2024-09-20T22:12:07.994 INFO:teuthology.orchestra.run.smithi037.stdout: Birth: 2024-09-20 22:08:56.455788581 +0000 2024-09-20T22:12:07.995 DEBUG:teuthology.orchestra.run.smithi037:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-20T22:12:08.058 INFO:teuthology.orchestra.run.smithi037.stderr:1+0 records in 2024-09-20T22:12:08.058 INFO:teuthology.orchestra.run.smithi037.stderr:1+0 records out 2024-09-20T22:12:08.058 INFO:teuthology.orchestra.run.smithi037.stderr:512 bytes copied, 0.000207236 s, 2.5 MB/s 2024-09-20T22:12:08.059 DEBUG:teuthology.orchestra.run.smithi037:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-20T22:12:08.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:07 smithi194 ceph-mon[25957]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T22:12:08.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:07 smithi194 ceph-mon[25957]: from='client.? 172.21.15.37:0/2598899264' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:12:08.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:07 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:12:08.116 DEBUG:teuthology.orchestra.run.smithi037:> stat /dev/vg_nvme/lv_3 2024-09-20T22:12:08.173 INFO:teuthology.orchestra.run.smithi037.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-20T22:12:08.173 INFO:teuthology.orchestra.run.smithi037.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-20T22:12:08.173 INFO:teuthology.orchestra.run.smithi037.stdout:Device: 5h/5d Inode: 705 Links: 1 2024-09-20T22:12:08.173 INFO:teuthology.orchestra.run.smithi037.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-20T22:12:08.173 INFO:teuthology.orchestra.run.smithi037.stdout:Context: system_u:object_r:device_t:s0 2024-09-20T22:12:08.173 INFO:teuthology.orchestra.run.smithi037.stdout:Access: 2024-09-20 22:11:28.495937388 +0000 2024-09-20T22:12:08.173 INFO:teuthology.orchestra.run.smithi037.stdout:Modify: 2024-09-20 22:08:56.782783764 +0000 2024-09-20T22:12:08.174 INFO:teuthology.orchestra.run.smithi037.stdout:Change: 2024-09-20 22:08:56.782783764 +0000 2024-09-20T22:12:08.174 INFO:teuthology.orchestra.run.smithi037.stdout: Birth: 2024-09-20 22:08:56.782783764 +0000 2024-09-20T22:12:08.174 DEBUG:teuthology.orchestra.run.smithi037:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-20T22:12:08.237 INFO:teuthology.orchestra.run.smithi037.stderr:1+0 records in 2024-09-20T22:12:08.238 INFO:teuthology.orchestra.run.smithi037.stderr:1+0 records out 2024-09-20T22:12:08.238 INFO:teuthology.orchestra.run.smithi037.stderr:512 bytes copied, 0.000211539 s, 2.4 MB/s 2024-09-20T22:12:08.239 DEBUG:teuthology.orchestra.run.smithi037:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-20T22:12:08.296 DEBUG:teuthology.orchestra.run.smithi037:> stat /dev/vg_nvme/lv_4 2024-09-20T22:12:08.351 INFO:teuthology.orchestra.run.smithi037.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-20T22:12:08.352 INFO:teuthology.orchestra.run.smithi037.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-20T22:12:08.352 INFO:teuthology.orchestra.run.smithi037.stdout:Device: 5h/5d Inode: 723 Links: 1 2024-09-20T22:12:08.352 INFO:teuthology.orchestra.run.smithi037.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-20T22:12:08.352 INFO:teuthology.orchestra.run.smithi037.stdout:Context: system_u:object_r:device_t:s0 2024-09-20T22:12:08.352 INFO:teuthology.orchestra.run.smithi037.stdout:Access: 2024-09-20 22:11:28.495937388 +0000 2024-09-20T22:12:08.352 INFO:teuthology.orchestra.run.smithi037.stdout:Modify: 2024-09-20 22:08:57.128778667 +0000 2024-09-20T22:12:08.352 INFO:teuthology.orchestra.run.smithi037.stdout:Change: 2024-09-20 22:08:57.128778667 +0000 2024-09-20T22:12:08.352 INFO:teuthology.orchestra.run.smithi037.stdout: Birth: 2024-09-20 22:08:57.128778667 +0000 2024-09-20T22:12:08.352 DEBUG:teuthology.orchestra.run.smithi037:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-20T22:12:08.417 INFO:teuthology.orchestra.run.smithi037.stderr:1+0 records in 2024-09-20T22:12:08.417 INFO:teuthology.orchestra.run.smithi037.stderr:1+0 records out 2024-09-20T22:12:08.417 INFO:teuthology.orchestra.run.smithi037.stderr:512 bytes copied, 0.000199325 s, 2.6 MB/s 2024-09-20T22:12:08.418 DEBUG:teuthology.orchestra.run.smithi037:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-20T22:12:08.475 DEBUG:teuthology.orchestra.run.smithi194:> set -ex 2024-09-20T22:12:08.475 DEBUG:teuthology.orchestra.run.smithi194:> dd if=/scratch_devs of=/dev/stdout 2024-09-20T22:12:08.494 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-20T22:12:08.494 DEBUG:teuthology.orchestra.run.smithi194:> stat /dev/vg_nvme/lv_1 2024-09-20T22:12:08.552 INFO:teuthology.orchestra.run.smithi194.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-20T22:12:08.552 INFO:teuthology.orchestra.run.smithi194.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-20T22:12:08.553 INFO:teuthology.orchestra.run.smithi194.stdout:Device: 5h/5d Inode: 667 Links: 1 2024-09-20T22:12:08.553 INFO:teuthology.orchestra.run.smithi194.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-20T22:12:08.553 INFO:teuthology.orchestra.run.smithi194.stdout:Context: system_u:object_r:device_t:s0 2024-09-20T22:12:08.553 INFO:teuthology.orchestra.run.smithi194.stdout:Access: 2024-09-20 22:12:04.158266641 +0000 2024-09-20T22:12:08.553 INFO:teuthology.orchestra.run.smithi194.stdout:Modify: 2024-09-20 22:08:49.542627983 +0000 2024-09-20T22:12:08.553 INFO:teuthology.orchestra.run.smithi194.stdout:Change: 2024-09-20 22:08:49.542627983 +0000 2024-09-20T22:12:08.553 INFO:teuthology.orchestra.run.smithi194.stdout: Birth: 2024-09-20 22:08:49.542627983 +0000 2024-09-20T22:12:08.553 DEBUG:teuthology.orchestra.run.smithi194:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-20T22:12:08.620 INFO:teuthology.orchestra.run.smithi194.stderr:1+0 records in 2024-09-20T22:12:08.620 INFO:teuthology.orchestra.run.smithi194.stderr:1+0 records out 2024-09-20T22:12:08.620 INFO:teuthology.orchestra.run.smithi194.stderr:512 bytes copied, 0.000287694 s, 1.8 MB/s 2024-09-20T22:12:08.622 DEBUG:teuthology.orchestra.run.smithi194:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-20T22:12:08.679 DEBUG:teuthology.orchestra.run.smithi194:> stat /dev/vg_nvme/lv_2 2024-09-20T22:12:08.736 INFO:teuthology.orchestra.run.smithi194.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-20T22:12:08.736 INFO:teuthology.orchestra.run.smithi194.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-20T22:12:08.737 INFO:teuthology.orchestra.run.smithi194.stdout:Device: 5h/5d Inode: 687 Links: 1 2024-09-20T22:12:08.737 INFO:teuthology.orchestra.run.smithi194.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-20T22:12:08.737 INFO:teuthology.orchestra.run.smithi194.stdout:Context: system_u:object_r:device_t:s0 2024-09-20T22:12:08.737 INFO:teuthology.orchestra.run.smithi194.stdout:Access: 2024-09-20 22:12:04.159266620 +0000 2024-09-20T22:12:08.737 INFO:teuthology.orchestra.run.smithi194.stdout:Modify: 2024-09-20 22:08:49.862621422 +0000 2024-09-20T22:12:08.737 INFO:teuthology.orchestra.run.smithi194.stdout:Change: 2024-09-20 22:08:49.862621422 +0000 2024-09-20T22:12:08.737 INFO:teuthology.orchestra.run.smithi194.stdout: Birth: 2024-09-20 22:08:49.862621422 +0000 2024-09-20T22:12:08.737 DEBUG:teuthology.orchestra.run.smithi194:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-20T22:12:08.802 INFO:teuthology.orchestra.run.smithi194.stderr:1+0 records in 2024-09-20T22:12:08.802 INFO:teuthology.orchestra.run.smithi194.stderr:1+0 records out 2024-09-20T22:12:08.802 INFO:teuthology.orchestra.run.smithi194.stderr:512 bytes copied, 0.000229687 s, 2.2 MB/s 2024-09-20T22:12:08.803 DEBUG:teuthology.orchestra.run.smithi194:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-20T22:12:08.861 DEBUG:teuthology.orchestra.run.smithi194:> stat /dev/vg_nvme/lv_3 2024-09-20T22:12:08.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:08 smithi037 ceph-mon[23718]: Deploying daemon prometheus.smithi037 on smithi037 2024-09-20T22:12:08.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:08 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:12:08.917 INFO:teuthology.orchestra.run.smithi194.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-20T22:12:08.917 INFO:teuthology.orchestra.run.smithi194.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-20T22:12:08.917 INFO:teuthology.orchestra.run.smithi194.stdout:Device: 5h/5d Inode: 703 Links: 1 2024-09-20T22:12:08.917 INFO:teuthology.orchestra.run.smithi194.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-20T22:12:08.917 INFO:teuthology.orchestra.run.smithi194.stdout:Context: system_u:object_r:device_t:s0 2024-09-20T22:12:08.917 INFO:teuthology.orchestra.run.smithi194.stdout:Access: 2024-09-20 22:12:04.159266620 +0000 2024-09-20T22:12:08.917 INFO:teuthology.orchestra.run.smithi194.stdout:Modify: 2024-09-20 22:08:50.185614799 +0000 2024-09-20T22:12:08.917 INFO:teuthology.orchestra.run.smithi194.stdout:Change: 2024-09-20 22:08:50.185614799 +0000 2024-09-20T22:12:08.917 INFO:teuthology.orchestra.run.smithi194.stdout: Birth: 2024-09-20 22:08:50.185614799 +0000 2024-09-20T22:12:08.917 DEBUG:teuthology.orchestra.run.smithi194:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-20T22:12:08.978 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:08 smithi194 ceph-mon[25957]: Deploying daemon prometheus.smithi037 on smithi037 2024-09-20T22:12:08.978 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:08 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:12:08.980 INFO:teuthology.orchestra.run.smithi194.stderr:1+0 records in 2024-09-20T22:12:08.980 INFO:teuthology.orchestra.run.smithi194.stderr:1+0 records out 2024-09-20T22:12:08.981 INFO:teuthology.orchestra.run.smithi194.stderr:512 bytes copied, 0.000212137 s, 2.4 MB/s 2024-09-20T22:12:08.982 DEBUG:teuthology.orchestra.run.smithi194:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-20T22:12:09.038 DEBUG:teuthology.orchestra.run.smithi194:> stat /dev/vg_nvme/lv_4 2024-09-20T22:12:09.093 INFO:teuthology.orchestra.run.smithi194.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-20T22:12:09.094 INFO:teuthology.orchestra.run.smithi194.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-20T22:12:09.094 INFO:teuthology.orchestra.run.smithi194.stdout:Device: 5h/5d Inode: 723 Links: 1 2024-09-20T22:12:09.094 INFO:teuthology.orchestra.run.smithi194.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-20T22:12:09.094 INFO:teuthology.orchestra.run.smithi194.stdout:Context: system_u:object_r:device_t:s0 2024-09-20T22:12:09.094 INFO:teuthology.orchestra.run.smithi194.stdout:Access: 2024-09-20 22:12:04.159266620 +0000 2024-09-20T22:12:09.094 INFO:teuthology.orchestra.run.smithi194.stdout:Modify: 2024-09-20 22:08:50.485608648 +0000 2024-09-20T22:12:09.094 INFO:teuthology.orchestra.run.smithi194.stdout:Change: 2024-09-20 22:08:50.485608648 +0000 2024-09-20T22:12:09.094 INFO:teuthology.orchestra.run.smithi194.stdout: Birth: 2024-09-20 22:08:50.485608648 +0000 2024-09-20T22:12:09.094 DEBUG:teuthology.orchestra.run.smithi194:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-20T22:12:09.157 INFO:teuthology.orchestra.run.smithi194.stderr:1+0 records in 2024-09-20T22:12:09.157 INFO:teuthology.orchestra.run.smithi194.stderr:1+0 records out 2024-09-20T22:12:09.157 INFO:teuthology.orchestra.run.smithi194.stderr:512 bytes copied, 0.000174265 s, 2.9 MB/s 2024-09-20T22:12:09.158 DEBUG:teuthology.orchestra.run.smithi194:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-20T22:12:09.214 INFO:tasks.cephadm:Deploying osd.0 on smithi037 with /dev/vg_nvme/lv_4... 2024-09-20T22:12:09.214 DEBUG:teuthology.orchestra.run.smithi037:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -- lvm zap /dev/vg_nvme/lv_4 2024-09-20T22:12:09.898 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:09 smithi037 ceph-mon[23718]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T22:12:10.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:09 smithi194 ceph-mon[25957]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T22:12:11.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:11 smithi037 ceph-mon[23718]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T22:12:12.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:11 smithi194 ceph-mon[25957]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T22:12:13.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:13 smithi037 ceph-mon[23718]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T22:12:14.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:13 smithi194 ceph-mon[25957]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T22:12:15.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:15 smithi037 ceph-mon[23718]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T22:12:16.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:15 smithi194 ceph-mon[25957]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T22:12:17.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:17 smithi037 ceph-mon[23718]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T22:12:18.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:17 smithi194 ceph-mon[25957]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T22:12:19.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:19 smithi194 ceph-mon[25957]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T22:12:19.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:19 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:12:19.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:19 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:12:19.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:19 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:12:19.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:19 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:12:19.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:19 smithi037 ceph-mon[23718]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T22:12:19.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:19 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:12:19.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:19 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:12:19.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:19 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:12:19.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:19 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:12:20.541 INFO:teuthology.orchestra.run.smithi037.stdout: 2024-09-20T22:12:20.560 DEBUG:teuthology.orchestra.run.smithi037:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -- ceph orch daemon add osd smithi037:vg_nvme/lv_4 2024-09-20T22:12:21.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:20 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:12:21.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:20 smithi037 ceph-mon[23718]: Updating smithi194:/etc/ceph/ceph.conf 2024-09-20T22:12:21.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:20 smithi037 ceph-mon[23718]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T22:12:21.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:20 smithi037 ceph-mon[23718]: Updating smithi194:/etc/ceph/ceph.client.admin.keyring 2024-09-20T22:12:21.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:20 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:12:21.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:20 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:12:21.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:20 smithi194 ceph-mon[25957]: Updating smithi194:/etc/ceph/ceph.conf 2024-09-20T22:12:21.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:20 smithi194 ceph-mon[25957]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T22:12:21.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:20 smithi194 ceph-mon[25957]: Updating smithi194:/etc/ceph/ceph.client.admin.keyring 2024-09-20T22:12:21.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:20 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:12:23.268 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:23 smithi037 ceph-mon[23718]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T22:12:23.268 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:23 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:12:23.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:23 smithi194 ceph-mon[25957]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T22:12:23.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:23 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:12:24.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:24 smithi037 ceph-mon[23718]: from='client.14238 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi037:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:12:24.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:24 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-20T22:12:24.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:24 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-20T22:12:24.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:24 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:12:24.495 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:24 smithi194 ceph-mon[25957]: from='client.14238 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi037:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:12:24.496 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:24 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-20T22:12:24.496 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:24 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-20T22:12:24.496 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:24 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:12:25.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:25 smithi037 ceph-mon[23718]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T22:12:26.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:25 smithi194 ceph-mon[25957]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T22:12:27.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:27 smithi037 ceph-mon[23718]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T22:12:27.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:27 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:12:27.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:27 smithi037 ceph-mon[23718]: from='client.? 172.21.15.37:0/1654235911' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "03a32e66-ac08-4daf-94c6-cd32f709392f"}]: dispatch 2024-09-20T22:12:27.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:27 smithi037 ceph-mon[23718]: from='client.? 172.21.15.37:0/1654235911' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "03a32e66-ac08-4daf-94c6-cd32f709392f"}]': finished 2024-09-20T22:12:27.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:27 smithi037 ceph-mon[23718]: osdmap e6: 1 total, 0 up, 1 in 2024-09-20T22:12:27.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:27 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-20T22:12:28.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:27 smithi194 ceph-mon[25957]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T22:12:28.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:27 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:12:28.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:27 smithi194 ceph-mon[25957]: from='client.? 172.21.15.37:0/1654235911' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "03a32e66-ac08-4daf-94c6-cd32f709392f"}]: dispatch 2024-09-20T22:12:28.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:27 smithi194 ceph-mon[25957]: from='client.? 172.21.15.37:0/1654235911' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "03a32e66-ac08-4daf-94c6-cd32f709392f"}]': finished 2024-09-20T22:12:28.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:27 smithi194 ceph-mon[25957]: osdmap e6: 1 total, 0 up, 1 in 2024-09-20T22:12:28.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:27 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-20T22:12:28.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:28 smithi037 ceph-mon[23718]: Updating smithi037:/etc/ceph/ceph.conf 2024-09-20T22:12:28.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:28 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:12:28.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:28 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:12:28.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:28 smithi037 ceph-mon[23718]: Reconfiguring mon.smithi037 (unknown last config time)... 2024-09-20T22:12:28.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:28 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-20T22:12:28.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:28 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-20T22:12:28.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:28 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:12:28.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:28 smithi037 ceph-mon[23718]: Reconfiguring daemon mon.smithi037 on smithi037 2024-09-20T22:12:28.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:28 smithi037 ceph-mon[23718]: from='client.? 172.21.15.37:0/3698549747' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-20T22:12:29.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:28 smithi194 ceph-mon[25957]: Updating smithi037:/etc/ceph/ceph.conf 2024-09-20T22:12:29.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:28 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:12:29.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:28 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:12:29.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:28 smithi194 ceph-mon[25957]: Reconfiguring mon.smithi037 (unknown last config time)... 2024-09-20T22:12:29.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:28 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-20T22:12:29.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:28 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-20T22:12:29.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:28 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:12:29.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:28 smithi194 ceph-mon[25957]: Reconfiguring daemon mon.smithi037 on smithi037 2024-09-20T22:12:29.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:28 smithi194 ceph-mon[25957]: from='client.? 172.21.15.37:0/3698549747' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-20T22:12:29.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:29 smithi194 ceph-mon[25957]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T22:12:30.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:29 smithi037 ceph-mon[23718]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T22:12:31.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:31 smithi037 ceph-mon[23718]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T22:12:32.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:31 smithi194 ceph-mon[25957]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T22:12:33.412 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:33 smithi037 ceph-mon[23718]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T22:12:33.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:33 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:12:33.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:33 smithi037 ceph-mon[23718]: Reconfiguring mgr.smithi037.lykblh (unknown last config time)... 2024-09-20T22:12:33.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:33 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi037.lykblh", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-20T22:12:33.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:33 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-20T22:12:33.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:33 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:12:33.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:33 smithi037 ceph-mon[23718]: Reconfiguring daemon mgr.smithi037.lykblh on smithi037 2024-09-20T22:12:33.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:33 smithi194 ceph-mon[25957]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T22:12:33.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:33 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:12:33.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:33 smithi194 ceph-mon[25957]: Reconfiguring mgr.smithi037.lykblh (unknown last config time)... 2024-09-20T22:12:33.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:33 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi037.lykblh", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-20T22:12:33.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:33 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-20T22:12:33.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:33 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:12:33.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:33 smithi194 ceph-mon[25957]: Reconfiguring daemon mgr.smithi037.lykblh on smithi037 2024-09-20T22:12:35.909 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:35 smithi037 ceph-mon[23718]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T22:12:35.909 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:35 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:12:35.909 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:35 smithi037 ceph-mon[23718]: Reconfiguring alertmanager.smithi037 (dependencies changed)... 2024-09-20T22:12:35.909 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:35 smithi037 ceph-mon[23718]: Reconfiguring daemon alertmanager.smithi037 on smithi037 2024-09-20T22:12:36.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:35 smithi194 ceph-mon[25957]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T22:12:36.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:35 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:12:36.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:35 smithi194 ceph-mon[25957]: Reconfiguring alertmanager.smithi037 (dependencies changed)... 2024-09-20T22:12:36.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:35 smithi194 ceph-mon[25957]: Reconfiguring daemon alertmanager.smithi037 on smithi037 2024-09-20T22:12:36.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:36 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-20T22:12:36.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:36 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:12:37.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:36 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-20T22:12:37.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:36 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:12:37.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:37 smithi037 ceph-mon[23718]: Deploying daemon osd.0 on smithi037 2024-09-20T22:12:37.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:37 smithi037 ceph-mon[23718]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T22:12:38.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:37 smithi194 ceph-mon[25957]: Deploying daemon osd.0 on smithi037 2024-09-20T22:12:38.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:37 smithi194 ceph-mon[25957]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T22:12:38.828 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:38 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi037.lykblh/mirror_snapshot_schedule"}]: dispatch 2024-09-20T22:12:38.829 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:38 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi037.lykblh/trash_purge_schedule"}]: dispatch 2024-09-20T22:12:39.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:38 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi037.lykblh/mirror_snapshot_schedule"}]: dispatch 2024-09-20T22:12:39.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:38 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi037.lykblh/trash_purge_schedule"}]: dispatch 2024-09-20T22:12:39.709 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:39 smithi037 ceph-mon[23718]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T22:12:39.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:39 smithi194 ceph-mon[25957]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T22:12:41.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:41 smithi037 ceph-mon[23718]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T22:12:41.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:41 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:12:41.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:41 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi037", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-20T22:12:41.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:41 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:12:42.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:41 smithi194 ceph-mon[25957]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T22:12:42.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:41 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:12:42.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:41 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi037", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-20T22:12:42.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:41 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:12:43.052 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:42 smithi037 ceph-mon[23718]: Reconfiguring crash.smithi037 (monmap changed)... 2024-09-20T22:12:43.053 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:42 smithi037 ceph-mon[23718]: Reconfiguring daemon crash.smithi037 on smithi037 2024-09-20T22:12:43.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:42 smithi194 ceph-mon[25957]: Reconfiguring crash.smithi037 (monmap changed)... 2024-09-20T22:12:43.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:42 smithi194 ceph-mon[25957]: Reconfiguring daemon crash.smithi037 on smithi037 2024-09-20T22:12:43.991 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:43 smithi037 ceph-mon[23718]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T22:12:44.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:43 smithi194 ceph-mon[25957]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T22:12:45.677 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:45 smithi037 ceph-mon[23718]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T22:12:46.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:45 smithi194 ceph-mon[25957]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T22:12:47.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:47 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:12:47.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:47 smithi037 ceph-mon[23718]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T22:12:47.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:47 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:12:47.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:47 smithi194 ceph-mon[25957]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T22:12:49.081 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:48 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:12:49.081 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:48 smithi037 ceph-mon[23718]: Reconfiguring grafana.smithi037 (dependencies changed)... 2024-09-20T22:12:49.081 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:48 smithi037 ceph-mon[23718]: Reconfiguring daemon grafana.smithi037 on smithi037 2024-09-20T22:12:49.081 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:48 smithi037 ceph-mon[23718]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T22:12:49.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:48 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:12:49.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:48 smithi194 ceph-mon[25957]: Reconfiguring grafana.smithi037 (dependencies changed)... 2024-09-20T22:12:49.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:48 smithi194 ceph-mon[25957]: Reconfiguring daemon grafana.smithi037 on smithi037 2024-09-20T22:12:49.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:48 smithi194 ceph-mon[25957]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T22:12:49.820 INFO:teuthology.orchestra.run.smithi037.stdout:Created osd(s) 0 on host 'smithi037' 2024-09-20T22:12:50.138 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:49 smithi037 ceph-mon[23718]: from='osd.0 [v2:172.21.15.37:6802/802225310,v1:172.21.15.37:6803/802225310]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-20T22:12:50.138 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:49 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:12:50.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:49 smithi194 ceph-mon[25957]: from='osd.0 [v2:172.21.15.37:6802/802225310,v1:172.21.15.37:6803/802225310]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-20T22:12:50.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:49 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:12:50.576 DEBUG:teuthology.orchestra.run.smithi037:osd.0> sudo journalctl -f -n 0 -u ceph-f9a6357a-779c-11ef-baf5-efdc52797490@osd.0.service 2024-09-20T22:12:50.578 INFO:tasks.cephadm:Deploying osd.1 on smithi037 with /dev/vg_nvme/lv_3... 2024-09-20T22:12:50.578 DEBUG:teuthology.orchestra.run.smithi037:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -- lvm zap /dev/vg_nvme/lv_3 2024-09-20T22:12:51.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:50 smithi037 ceph-mon[23718]: from='osd.0 [v2:172.21.15.37:6802/802225310,v1:172.21.15.37:6803/802225310]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-20T22:12:51.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:50 smithi037 ceph-mon[23718]: osdmap e7: 1 total, 0 up, 1 in 2024-09-20T22:12:51.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:50 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-20T22:12:51.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:50 smithi037 ceph-mon[23718]: from='osd.0 [v2:172.21.15.37:6802/802225310,v1:172.21.15.37:6803/802225310]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi037", "root=default"]}]: dispatch 2024-09-20T22:12:51.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:50 smithi037 ceph-mon[23718]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T22:12:51.163 INFO:journalctl@ceph.osd.0.smithi037.stdout:Sep 20 22:12:50 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-0[34232]: 2024-09-20T22:12:50.888+0000 7fef6f961700 -1 osd.0 0 waiting for initial osdmap 2024-09-20T22:12:51.163 INFO:journalctl@ceph.osd.0.smithi037.stdout:Sep 20 22:12:50 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-0[34232]: 2024-09-20T22:12:50.890+0000 7fef68af5700 -1 osd.0 8 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-20T22:12:51.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:50 smithi194 ceph-mon[25957]: from='osd.0 [v2:172.21.15.37:6802/802225310,v1:172.21.15.37:6803/802225310]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-20T22:12:51.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:50 smithi194 ceph-mon[25957]: osdmap e7: 1 total, 0 up, 1 in 2024-09-20T22:12:51.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:50 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-20T22:12:51.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:50 smithi194 ceph-mon[25957]: from='osd.0 [v2:172.21.15.37:6802/802225310,v1:172.21.15.37:6803/802225310]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi037", "root=default"]}]: dispatch 2024-09-20T22:12:51.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:50 smithi194 ceph-mon[25957]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T22:12:52.090 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:51 smithi037 ceph-mon[23718]: from='osd.0 [v2:172.21.15.37:6802/802225310,v1:172.21.15.37:6803/802225310]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi037", "root=default"]}]': finished 2024-09-20T22:12:52.090 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:51 smithi037 ceph-mon[23718]: osdmap e8: 1 total, 0 up, 1 in 2024-09-20T22:12:52.090 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:51 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-20T22:12:52.090 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:51 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-20T22:12:52.090 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:51 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-20T22:12:52.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:51 smithi194 ceph-mon[25957]: from='osd.0 [v2:172.21.15.37:6802/802225310,v1:172.21.15.37:6803/802225310]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi037", "root=default"]}]': finished 2024-09-20T22:12:52.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:51 smithi194 ceph-mon[25957]: osdmap e8: 1 total, 0 up, 1 in 2024-09-20T22:12:52.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:51 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-20T22:12:52.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:51 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-20T22:12:52.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:51 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-20T22:12:53.095 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:52 smithi037 ceph-mon[23718]: purged_snaps scrub starts 2024-09-20T22:12:53.096 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:52 smithi037 ceph-mon[23718]: purged_snaps scrub ok 2024-09-20T22:12:53.096 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:52 smithi037 ceph-mon[23718]: osd.0 [v2:172.21.15.37:6802/802225310,v1:172.21.15.37:6803/802225310] boot 2024-09-20T22:12:53.096 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:52 smithi037 ceph-mon[23718]: osdmap e9: 1 total, 1 up, 1 in 2024-09-20T22:12:53.096 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:52 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-20T22:12:53.096 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:52 smithi037 ceph-mon[23718]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T22:12:53.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:52 smithi194 ceph-mon[25957]: purged_snaps scrub starts 2024-09-20T22:12:53.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:52 smithi194 ceph-mon[25957]: purged_snaps scrub ok 2024-09-20T22:12:53.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:52 smithi194 ceph-mon[25957]: osd.0 [v2:172.21.15.37:6802/802225310,v1:172.21.15.37:6803/802225310] boot 2024-09-20T22:12:53.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:52 smithi194 ceph-mon[25957]: osdmap e9: 1 total, 1 up, 1 in 2024-09-20T22:12:53.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:52 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-20T22:12:53.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:52 smithi194 ceph-mon[25957]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T22:12:54.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:53 smithi037 ceph-mon[23718]: osdmap e10: 1 total, 1 up, 1 in 2024-09-20T22:12:54.245 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:53 smithi194 ceph-mon[25957]: osdmap e10: 1 total, 1 up, 1 in 2024-09-20T22:12:55.030 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:54 smithi037 ceph-mon[23718]: pgmap v36: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-20T22:12:55.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:54 smithi194 ceph-mon[25957]: pgmap v36: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-20T22:12:56.591 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:56 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:12:56.591 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:56 smithi037 ceph-mon[23718]: Reconfiguring crash.smithi194 (monmap changed)... 2024-09-20T22:12:56.591 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:56 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi194", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-20T22:12:56.591 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:56 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:12:56.591 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:56 smithi037 ceph-mon[23718]: Reconfiguring daemon crash.smithi194 on smithi194 2024-09-20T22:12:56.753 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:56 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:12:56.753 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:56 smithi194 ceph-mon[25957]: Reconfiguring crash.smithi194 (monmap changed)... 2024-09-20T22:12:56.753 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:56 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi194", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-20T22:12:56.754 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:56 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:12:56.754 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:56 smithi194 ceph-mon[25957]: Reconfiguring daemon crash.smithi194 on smithi194 2024-09-20T22:12:56.863 INFO:teuthology.orchestra.run.smithi037.stdout: 2024-09-20T22:12:56.886 DEBUG:teuthology.orchestra.run.smithi037:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -- ceph orch daemon add osd smithi037:vg_nvme/lv_3 2024-09-20T22:12:57.526 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:57 smithi037 ceph-mon[23718]: pgmap v37: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-20T22:12:57.526 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:57 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:12:57.526 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:57 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi194.bkcglt", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-20T22:12:57.526 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:57 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-20T22:12:57.526 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:57 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:12:57.818 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:57 smithi194 ceph-mon[25957]: pgmap v37: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-20T22:12:57.818 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:57 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:12:57.818 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:57 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi194.bkcglt", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-20T22:12:57.818 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:57 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-20T22:12:57.818 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:57 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:12:58.444 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:58 smithi037 ceph-mon[23718]: Reconfiguring mgr.smithi194.bkcglt (monmap changed)... 2024-09-20T22:12:58.444 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:58 smithi037 ceph-mon[23718]: Reconfiguring daemon mgr.smithi194.bkcglt on smithi194 2024-09-20T22:12:58.444 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:58 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:12:58.444 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:58 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-20T22:12:58.444 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:58 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-20T22:12:58.444 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:58 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:12:58.754 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:58 smithi194 ceph-mon[25957]: Reconfiguring mgr.smithi194.bkcglt (monmap changed)... 2024-09-20T22:12:58.754 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:58 smithi194 ceph-mon[25957]: Reconfiguring daemon mgr.smithi194.bkcglt on smithi194 2024-09-20T22:12:58.754 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:58 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:12:58.754 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:58 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-20T22:12:58.754 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:58 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-20T22:12:58.754 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:58 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:12:59.831 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:59 smithi037 ceph-mon[23718]: pgmap v38: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-20T22:12:59.831 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:59 smithi037 ceph-mon[23718]: Reconfiguring mon.smithi194 (monmap changed)... 2024-09-20T22:12:59.832 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:59 smithi037 ceph-mon[23718]: Reconfiguring daemon mon.smithi194 on smithi194 2024-09-20T22:12:59.832 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:59 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-20T22:12:59.832 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:59 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-20T22:12:59.832 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:59 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:12:59.832 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:59 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:12:59.832 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:12:59 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-20T22:13:00.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:59 smithi194 ceph-mon[25957]: pgmap v38: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-20T22:13:00.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:59 smithi194 ceph-mon[25957]: Reconfiguring mon.smithi194 (monmap changed)... 2024-09-20T22:13:00.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:59 smithi194 ceph-mon[25957]: Reconfiguring daemon mon.smithi194 on smithi194 2024-09-20T22:13:00.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:59 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-20T22:13:00.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:59 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-20T22:13:00.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:59 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:13:00.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:59 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:13:00.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:12:59 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-20T22:13:00.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:00 smithi037 ceph-mon[23718]: from='client.14256 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi037:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:13:00.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:00 smithi037 ceph-mon[23718]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-20T22:13:01.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:00 smithi194 ceph-mon[25957]: from='client.14256 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi037:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:13:01.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:00 smithi194 ceph-mon[25957]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-20T22:13:01.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:01 smithi037 ceph-mon[23718]: pgmap v39: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-20T22:13:01.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:01 smithi037 ceph-mon[23718]: from='client.? 172.21.15.37:0/1486852934' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5f6c77dd-46a1-4dd2-91e0-f05faa539e5a"}]: dispatch 2024-09-20T22:13:01.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:01 smithi037 ceph-mon[23718]: from='client.? 172.21.15.37:0/1486852934' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "5f6c77dd-46a1-4dd2-91e0-f05faa539e5a"}]': finished 2024-09-20T22:13:01.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:01 smithi037 ceph-mon[23718]: osdmap e11: 2 total, 1 up, 2 in 2024-09-20T22:13:01.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:01 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-20T22:13:02.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:01 smithi194 ceph-mon[25957]: pgmap v39: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-20T22:13:02.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:01 smithi194 ceph-mon[25957]: from='client.? 172.21.15.37:0/1486852934' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5f6c77dd-46a1-4dd2-91e0-f05faa539e5a"}]: dispatch 2024-09-20T22:13:02.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:01 smithi194 ceph-mon[25957]: from='client.? 172.21.15.37:0/1486852934' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "5f6c77dd-46a1-4dd2-91e0-f05faa539e5a"}]': finished 2024-09-20T22:13:02.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:01 smithi194 ceph-mon[25957]: osdmap e11: 2 total, 1 up, 2 in 2024-09-20T22:13:02.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:01 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-20T22:13:02.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:02 smithi037 ceph-mon[23718]: from='client.? 172.21.15.37:0/342882443' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-20T22:13:03.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:02 smithi194 ceph-mon[25957]: from='client.? 172.21.15.37:0/342882443' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-20T22:13:03.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:03 smithi037 ceph-mon[23718]: pgmap v41: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-20T22:13:04.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:03 smithi194 ceph-mon[25957]: pgmap v41: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-20T22:13:05.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:05 smithi194 ceph-mon[25957]: pgmap v42: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-20T22:13:05.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:05 smithi037 ceph-mon[23718]: pgmap v42: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-20T22:13:07.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:07 smithi037 ceph-mon[23718]: pgmap v43: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-20T22:13:07.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:07 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-20T22:13:07.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:07 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:13:08.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:07 smithi194 ceph-mon[25957]: pgmap v43: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-20T22:13:08.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:07 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-20T22:13:08.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:07 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:13:08.827 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:08 smithi037 ceph-mon[23718]: Deploying daemon osd.1 on smithi037 2024-09-20T22:13:09.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:08 smithi194 ceph-mon[25957]: Deploying daemon osd.1 on smithi037 2024-09-20T22:13:09.816 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:09 smithi037 ceph-mon[23718]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-20T22:13:09.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:09 smithi194 ceph-mon[25957]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-20T22:13:11.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:11 smithi037 ceph-mon[23718]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-20T22:13:12.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:11 smithi194 ceph-mon[25957]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-20T22:13:14.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:13 smithi194 ceph-mon[25957]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-20T22:13:14.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:13 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:13:14.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:13 smithi037 ceph-mon[23718]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-20T22:13:14.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:13 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:13:16.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:15 smithi194 ceph-mon[25957]: pgmap v47: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-20T22:13:16.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:15 smithi037 ceph-mon[23718]: pgmap v47: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-20T22:13:16.334 INFO:teuthology.orchestra.run.smithi037.stdout:Created osd(s) 1 on host 'smithi037' 2024-09-20T22:13:16.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:16 smithi037 ceph-mon[23718]: from='osd.1 [v2:172.21.15.37:6810/441045715,v1:172.21.15.37:6811/441045715]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-20T22:13:16.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:16 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:13:16.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:16 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.37:9093"}]: dispatch 2024-09-20T22:13:16.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:16 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:13:16.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:16 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-20T22:13:16.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:16 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.37:3000"}]: dispatch 2024-09-20T22:13:16.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:16 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:13:16.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:16 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-20T22:13:16.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:16 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.37:9095"}]: dispatch 2024-09-20T22:13:16.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:16 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:13:16.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:16 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:13:16.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:16 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:13:16.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:16 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:13:16.978 DEBUG:teuthology.orchestra.run.smithi037:osd.1> sudo journalctl -f -n 0 -u ceph-f9a6357a-779c-11ef-baf5-efdc52797490@osd.1.service 2024-09-20T22:13:16.980 INFO:tasks.cephadm:Deploying osd.2 on smithi037 with /dev/vg_nvme/lv_2... 2024-09-20T22:13:16.981 DEBUG:teuthology.orchestra.run.smithi037:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -- lvm zap /dev/vg_nvme/lv_2 2024-09-20T22:13:16.993 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:16 smithi194 ceph-mon[25957]: from='osd.1 [v2:172.21.15.37:6810/441045715,v1:172.21.15.37:6811/441045715]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-20T22:13:16.993 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:16 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:13:16.993 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:16 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.37:9093"}]: dispatch 2024-09-20T22:13:16.993 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:16 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:13:16.993 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:16 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-20T22:13:16.993 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:16 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.37:3000"}]: dispatch 2024-09-20T22:13:16.993 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:16 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:13:16.993 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:16 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-20T22:13:16.993 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:16 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.37:9095"}]: dispatch 2024-09-20T22:13:16.993 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:16 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:13:16.993 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:16 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:13:16.993 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:16 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:13:16.993 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:16 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:13:17.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:17 smithi037 ceph-mon[23718]: pgmap v48: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-20T22:13:17.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:17 smithi037 ceph-mon[23718]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.37:9093"}]: dispatch 2024-09-20T22:13:17.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:17 smithi037 ceph-mon[23718]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-20T22:13:17.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:17 smithi037 ceph-mon[23718]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.37:3000"}]: dispatch 2024-09-20T22:13:17.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:17 smithi037 ceph-mon[23718]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-20T22:13:17.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:17 smithi037 ceph-mon[23718]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.37:9095"}]: dispatch 2024-09-20T22:13:17.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:17 smithi037 ceph-mon[23718]: from='osd.1 [v2:172.21.15.37:6810/441045715,v1:172.21.15.37:6811/441045715]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-20T22:13:17.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:17 smithi037 ceph-mon[23718]: osdmap e12: 2 total, 1 up, 2 in 2024-09-20T22:13:17.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:17 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-20T22:13:17.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:17 smithi037 ceph-mon[23718]: from='osd.1 [v2:172.21.15.37:6810/441045715,v1:172.21.15.37:6811/441045715]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi037", "root=default"]}]: dispatch 2024-09-20T22:13:17.913 INFO:journalctl@ceph.osd.1.smithi037.stdout:Sep 20 22:13:17 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-1[37075]: 2024-09-20T22:13:17.690+0000 7f7019384700 -1 osd.1 0 waiting for initial osdmap 2024-09-20T22:13:17.913 INFO:journalctl@ceph.osd.1.smithi037.stdout:Sep 20 22:13:17 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-1[37075]: 2024-09-20T22:13:17.693+0000 7f7012518700 -1 osd.1 13 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-20T22:13:18.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:17 smithi194 ceph-mon[25957]: pgmap v48: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-20T22:13:18.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:17 smithi194 ceph-mon[25957]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.37:9093"}]: dispatch 2024-09-20T22:13:18.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:17 smithi194 ceph-mon[25957]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-20T22:13:18.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:17 smithi194 ceph-mon[25957]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.37:3000"}]: dispatch 2024-09-20T22:13:18.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:17 smithi194 ceph-mon[25957]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-20T22:13:18.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:17 smithi194 ceph-mon[25957]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.37:9095"}]: dispatch 2024-09-20T22:13:18.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:17 smithi194 ceph-mon[25957]: from='osd.1 [v2:172.21.15.37:6810/441045715,v1:172.21.15.37:6811/441045715]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-20T22:13:18.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:17 smithi194 ceph-mon[25957]: osdmap e12: 2 total, 1 up, 2 in 2024-09-20T22:13:18.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:17 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-20T22:13:18.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:17 smithi194 ceph-mon[25957]: from='osd.1 [v2:172.21.15.37:6810/441045715,v1:172.21.15.37:6811/441045715]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi037", "root=default"]}]: dispatch 2024-09-20T22:13:18.797 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:18 smithi037 ceph-mon[23718]: from='osd.1 [v2:172.21.15.37:6810/441045715,v1:172.21.15.37:6811/441045715]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi037", "root=default"]}]': finished 2024-09-20T22:13:18.797 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:18 smithi037 ceph-mon[23718]: osdmap e13: 2 total, 1 up, 2 in 2024-09-20T22:13:18.797 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:18 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-20T22:13:18.797 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:18 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-20T22:13:18.797 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:18 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:13:19.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:18 smithi194 ceph-mon[25957]: from='osd.1 [v2:172.21.15.37:6810/441045715,v1:172.21.15.37:6811/441045715]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi037", "root=default"]}]': finished 2024-09-20T22:13:19.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:18 smithi194 ceph-mon[25957]: osdmap e13: 2 total, 1 up, 2 in 2024-09-20T22:13:19.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:18 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-20T22:13:19.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:18 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-20T22:13:19.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:18 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:13:19.801 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:19 smithi037 ceph-mon[23718]: purged_snaps scrub starts 2024-09-20T22:13:19.801 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:19 smithi037 ceph-mon[23718]: purged_snaps scrub ok 2024-09-20T22:13:19.801 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:19 smithi037 ceph-mon[23718]: pgmap v51: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-20T22:13:19.801 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:19 smithi037 ceph-mon[23718]: osd.1 [v2:172.21.15.37:6810/441045715,v1:172.21.15.37:6811/441045715] boot 2024-09-20T22:13:19.801 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:19 smithi037 ceph-mon[23718]: osdmap e14: 2 total, 2 up, 2 in 2024-09-20T22:13:19.801 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:19 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-20T22:13:20.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:19 smithi194 ceph-mon[25957]: purged_snaps scrub starts 2024-09-20T22:13:20.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:19 smithi194 ceph-mon[25957]: purged_snaps scrub ok 2024-09-20T22:13:20.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:19 smithi194 ceph-mon[25957]: pgmap v51: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-20T22:13:20.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:19 smithi194 ceph-mon[25957]: osd.1 [v2:172.21.15.37:6810/441045715,v1:172.21.15.37:6811/441045715] boot 2024-09-20T22:13:20.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:19 smithi194 ceph-mon[25957]: osdmap e14: 2 total, 2 up, 2 in 2024-09-20T22:13:20.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:19 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-20T22:13:21.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:21 smithi037 ceph-mon[23718]: osdmap e15: 2 total, 2 up, 2 in 2024-09-20T22:13:21.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:21 smithi037 ceph-mon[23718]: pgmap v54: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-09-20T22:13:21.459 INFO:teuthology.orchestra.run.smithi037.stdout: 2024-09-20T22:13:21.480 DEBUG:teuthology.orchestra.run.smithi037:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -- ceph orch daemon add osd smithi037:vg_nvme/lv_2 2024-09-20T22:13:21.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:21 smithi194 ceph-mon[25957]: osdmap e15: 2 total, 2 up, 2 in 2024-09-20T22:13:21.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:21 smithi194 ceph-mon[25957]: pgmap v54: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-09-20T22:13:23.248 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:23 smithi037 ceph-mon[23718]: pgmap v55: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-09-20T22:13:23.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:23 smithi194 ceph-mon[25957]: pgmap v55: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-09-20T22:13:25.645 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:25 smithi037 ceph-mon[23718]: pgmap v56: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T22:13:25.646 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:25 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:13:25.646 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:25 smithi037 ceph-mon[23718]: from='client.14274 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi037:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:13:25.646 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:25 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-20T22:13:25.646 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:25 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-20T22:13:25.646 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:25 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:13:25.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:25 smithi194 ceph-mon[25957]: pgmap v56: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T22:13:25.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:25 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:13:25.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:25 smithi194 ceph-mon[25957]: from='client.14274 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi037:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:13:25.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:25 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-20T22:13:25.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:25 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-20T22:13:25.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:25 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:13:27.570 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:27 smithi037 ceph-mon[23718]: pgmap v57: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T22:13:27.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:27 smithi194 ceph-mon[25957]: pgmap v57: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T22:13:29.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:29 smithi194 ceph-mon[25957]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T22:13:29.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:29 smithi194 ceph-mon[25957]: Detected new or changed devices on smithi037 2024-09-20T22:13:29.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:29 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:13:29.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:29 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-20T22:13:29.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:29 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-20T22:13:29.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:29 smithi194 ceph-mon[25957]: Adjusting osd_memory_target on smithi037 to 6434M 2024-09-20T22:13:29.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:29 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:13:29.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:29 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:13:29.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:29 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:13:29.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:29 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:13:29.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:29 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:13:29.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:29 smithi037 ceph-mon[23718]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T22:13:29.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:29 smithi037 ceph-mon[23718]: Detected new or changed devices on smithi037 2024-09-20T22:13:29.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:29 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:13:29.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:29 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-20T22:13:29.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:29 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-20T22:13:29.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:29 smithi037 ceph-mon[23718]: Adjusting osd_memory_target on smithi037 to 6434M 2024-09-20T22:13:29.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:29 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:13:29.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:29 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:13:29.664 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:29 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:13:29.664 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:29 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:13:29.664 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:29 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:13:30.662 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:30 smithi037 ceph-mon[23718]: from='client.? 172.21.15.37:0/1794607429' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3469715b-d6bc-4535-9a87-ea486cd8fd9a"}]: dispatch 2024-09-20T22:13:30.662 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:30 smithi037 ceph-mon[23718]: from='client.? 172.21.15.37:0/1794607429' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3469715b-d6bc-4535-9a87-ea486cd8fd9a"}]': finished 2024-09-20T22:13:30.662 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:30 smithi037 ceph-mon[23718]: osdmap e16: 3 total, 2 up, 3 in 2024-09-20T22:13:30.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:30 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-20T22:13:30.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:30 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:13:30.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:30 smithi037 ceph-mon[23718]: from='client.? 172.21.15.37:0/3767672503' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-20T22:13:30.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:30 smithi194 ceph-mon[25957]: from='client.? 172.21.15.37:0/1794607429' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3469715b-d6bc-4535-9a87-ea486cd8fd9a"}]: dispatch 2024-09-20T22:13:30.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:30 smithi194 ceph-mon[25957]: from='client.? 172.21.15.37:0/1794607429' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3469715b-d6bc-4535-9a87-ea486cd8fd9a"}]': finished 2024-09-20T22:13:30.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:30 smithi194 ceph-mon[25957]: osdmap e16: 3 total, 2 up, 3 in 2024-09-20T22:13:30.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:30 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-20T22:13:30.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:30 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:13:30.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:30 smithi194 ceph-mon[25957]: from='client.? 172.21.15.37:0/3767672503' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-20T22:13:31.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:31 smithi194 ceph-mon[25957]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T22:13:31.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:31 smithi037 ceph-mon[23718]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T22:13:33.662 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:33 smithi037 ceph-mon[23718]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T22:13:33.662 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:33 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:13:33.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:33 smithi194 ceph-mon[25957]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T22:13:33.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:33 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:13:35.824 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:35 smithi037 ceph-mon[23718]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T22:13:36.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:35 smithi194 ceph-mon[25957]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T22:13:36.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:36 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-20T22:13:36.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:36 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:13:37.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:36 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-20T22:13:37.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:36 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:13:37.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:37 smithi194 ceph-mon[25957]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T22:13:37.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:37 smithi194 ceph-mon[25957]: Deploying daemon osd.2 on smithi037 2024-09-20T22:13:37.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:37 smithi037 ceph-mon[23718]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T22:13:37.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:37 smithi037 ceph-mon[23718]: Deploying daemon osd.2 on smithi037 2024-09-20T22:13:38.727 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:38 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi037.lykblh/mirror_snapshot_schedule"}]: dispatch 2024-09-20T22:13:38.728 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:38 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi037.lykblh/trash_purge_schedule"}]: dispatch 2024-09-20T22:13:39.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:38 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi037.lykblh/mirror_snapshot_schedule"}]: dispatch 2024-09-20T22:13:39.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:38 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi037.lykblh/trash_purge_schedule"}]: dispatch 2024-09-20T22:13:39.942 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:39 smithi037 ceph-mon[23718]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T22:13:39.942 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:39 smithi037 ceph-mon[23718]: Detected new or changed devices on smithi037 2024-09-20T22:13:39.942 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:39 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:13:39.942 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:39 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:13:40.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:39 smithi194 ceph-mon[25957]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T22:13:40.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:39 smithi194 ceph-mon[25957]: Detected new or changed devices on smithi037 2024-09-20T22:13:40.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:39 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:13:40.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:39 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:13:42.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:41 smithi194 ceph-mon[25957]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T22:13:42.103 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:41 smithi037 ceph-mon[23718]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T22:13:44.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:43 smithi194 ceph-mon[25957]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T22:13:44.139 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:43 smithi037 ceph-mon[23718]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T22:13:45.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:44 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:13:45.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:44 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:13:45.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:44 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:13:45.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:44 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:13:45.116 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:44 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:13:45.116 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:44 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:13:45.116 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:44 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:13:45.116 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:44 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:13:46.023 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:45 smithi037 ceph-mon[23718]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T22:13:46.023 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:45 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:13:46.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:45 smithi194 ceph-mon[25957]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T22:13:46.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:45 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:13:46.900 INFO:teuthology.orchestra.run.smithi037.stdout:Created osd(s) 2 on host 'smithi037' 2024-09-20T22:13:47.760 DEBUG:teuthology.orchestra.run.smithi037:osd.2> sudo journalctl -f -n 0 -u ceph-f9a6357a-779c-11ef-baf5-efdc52797490@osd.2.service 2024-09-20T22:13:47.763 INFO:tasks.cephadm:Deploying osd.3 on smithi037 with /dev/vg_nvme/lv_1... 2024-09-20T22:13:47.763 DEBUG:teuthology.orchestra.run.smithi037:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -- lvm zap /dev/vg_nvme/lv_1 2024-09-20T22:13:48.002 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:47 smithi037 ceph-mon[23718]: pgmap v68: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T22:13:48.002 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:47 smithi037 ceph-mon[23718]: from='osd.2 [v2:172.21.15.37:6818/4264091112,v1:172.21.15.37:6819/4264091112]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-20T22:13:48.002 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:47 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:13:48.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:47 smithi194 ceph-mon[25957]: pgmap v68: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T22:13:48.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:47 smithi194 ceph-mon[25957]: from='osd.2 [v2:172.21.15.37:6818/4264091112,v1:172.21.15.37:6819/4264091112]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-20T22:13:48.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:47 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:13:48.897 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:48 smithi037 ceph-mon[23718]: from='osd.2 [v2:172.21.15.37:6818/4264091112,v1:172.21.15.37:6819/4264091112]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-20T22:13:48.897 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:48 smithi037 ceph-mon[23718]: osdmap e17: 3 total, 2 up, 3 in 2024-09-20T22:13:48.897 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:48 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-20T22:13:48.897 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:48 smithi037 ceph-mon[23718]: from='osd.2 [v2:172.21.15.37:6818/4264091112,v1:172.21.15.37:6819/4264091112]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi037", "root=default"]}]: dispatch 2024-09-20T22:13:48.897 INFO:journalctl@ceph.osd.2.smithi037.stdout:Sep 20 22:13:48 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-2[40992]: 2024-09-20T22:13:48.790+0000 7f9fe6d3f700 -1 osd.2 0 waiting for initial osdmap 2024-09-20T22:13:48.897 INFO:journalctl@ceph.osd.2.smithi037.stdout:Sep 20 22:13:48 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-2[40992]: 2024-09-20T22:13:48.794+0000 7f9fe26d8700 -1 osd.2 18 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-20T22:13:49.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:48 smithi194 ceph-mon[25957]: from='osd.2 [v2:172.21.15.37:6818/4264091112,v1:172.21.15.37:6819/4264091112]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-20T22:13:49.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:48 smithi194 ceph-mon[25957]: osdmap e17: 3 total, 2 up, 3 in 2024-09-20T22:13:49.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:48 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-20T22:13:49.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:48 smithi194 ceph-mon[25957]: from='osd.2 [v2:172.21.15.37:6818/4264091112,v1:172.21.15.37:6819/4264091112]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi037", "root=default"]}]: dispatch 2024-09-20T22:13:49.861 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:49 smithi037 ceph-mon[23718]: pgmap v70: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T22:13:49.861 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:49 smithi037 ceph-mon[23718]: from='osd.2 [v2:172.21.15.37:6818/4264091112,v1:172.21.15.37:6819/4264091112]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi037", "root=default"]}]': finished 2024-09-20T22:13:49.861 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:49 smithi037 ceph-mon[23718]: osdmap e18: 3 total, 2 up, 3 in 2024-09-20T22:13:49.861 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:49 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-20T22:13:49.861 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:49 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-20T22:13:50.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:49 smithi194 ceph-mon[25957]: pgmap v70: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T22:13:50.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:49 smithi194 ceph-mon[25957]: from='osd.2 [v2:172.21.15.37:6818/4264091112,v1:172.21.15.37:6819/4264091112]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi037", "root=default"]}]': finished 2024-09-20T22:13:50.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:49 smithi194 ceph-mon[25957]: osdmap e18: 3 total, 2 up, 3 in 2024-09-20T22:13:50.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:49 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-20T22:13:50.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:49 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-20T22:13:51.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:50 smithi194 ceph-mon[25957]: purged_snaps scrub starts 2024-09-20T22:13:51.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:50 smithi194 ceph-mon[25957]: purged_snaps scrub ok 2024-09-20T22:13:51.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:50 smithi194 ceph-mon[25957]: osd.2 [v2:172.21.15.37:6818/4264091112,v1:172.21.15.37:6819/4264091112] boot 2024-09-20T22:13:51.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:50 smithi194 ceph-mon[25957]: osdmap e19: 3 total, 3 up, 3 in 2024-09-20T22:13:51.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:50 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-20T22:13:51.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:50 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-09-20T22:13:51.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:50 smithi037 ceph-mon[23718]: purged_snaps scrub starts 2024-09-20T22:13:51.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:50 smithi037 ceph-mon[23718]: purged_snaps scrub ok 2024-09-20T22:13:51.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:50 smithi037 ceph-mon[23718]: osd.2 [v2:172.21.15.37:6818/4264091112,v1:172.21.15.37:6819/4264091112] boot 2024-09-20T22:13:51.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:50 smithi037 ceph-mon[23718]: osdmap e19: 3 total, 3 up, 3 in 2024-09-20T22:13:51.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:50 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-20T22:13:51.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:50 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-09-20T22:13:51.233 INFO:teuthology.orchestra.run.smithi037.stdout: 2024-09-20T22:13:51.255 DEBUG:teuthology.orchestra.run.smithi037:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -- ceph orch daemon add osd smithi037:vg_nvme/lv_1 2024-09-20T22:13:52.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:51 smithi194 ceph-mon[25957]: pgmap v73: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2024-09-20T22:13:52.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:51 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-09-20T22:13:52.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:51 smithi194 ceph-mon[25957]: osdmap e20: 3 total, 3 up, 3 in 2024-09-20T22:13:52.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:51 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-20T22:13:52.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:51 smithi037 ceph-mon[23718]: pgmap v73: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2024-09-20T22:13:52.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:51 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-09-20T22:13:52.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:51 smithi037 ceph-mon[23718]: osdmap e20: 3 total, 3 up, 3 in 2024-09-20T22:13:52.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:51 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-20T22:13:52.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:52 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-20T22:13:52.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:52 smithi037 ceph-mon[23718]: osdmap e21: 3 total, 3 up, 3 in 2024-09-20T22:13:52.913 INFO:journalctl@ceph.osd.0.smithi037.stdout:Sep 20 22:13:52 smithi037 sudo[42300]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-20T22:13:52.913 INFO:journalctl@ceph.osd.0.smithi037.stdout:Sep 20 22:13:52 smithi037 sudo[42300]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-20T22:13:52.913 INFO:journalctl@ceph.osd.0.smithi037.stdout:Sep 20 22:13:52 smithi037 sudo[42300]: pam_unix(sudo:session): session closed for user root 2024-09-20T22:13:53.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:52 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-20T22:13:53.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:52 smithi194 ceph-mon[25957]: osdmap e21: 3 total, 3 up, 3 in 2024-09-20T22:13:53.323 INFO:journalctl@ceph.osd.0.smithi037.stdout:Sep 20 22:13:53 smithi037 sudo[42333]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-20T22:13:53.323 INFO:journalctl@ceph.osd.0.smithi037.stdout:Sep 20 22:13:53 smithi037 sudo[42333]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-20T22:13:53.323 INFO:journalctl@ceph.osd.0.smithi037.stdout:Sep 20 22:13:53 smithi037 sudo[42333]: pam_unix(sudo:session): session closed for user root 2024-09-20T22:13:53.631 INFO:journalctl@ceph.osd.1.smithi037.stdout:Sep 20 22:13:53 smithi037 sudo[42353]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-20T22:13:53.631 INFO:journalctl@ceph.osd.1.smithi037.stdout:Sep 20 22:13:53 smithi037 sudo[42353]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-20T22:13:53.631 INFO:journalctl@ceph.osd.1.smithi037.stdout:Sep 20 22:13:53 smithi037 sudo[42353]: pam_unix(sudo:session): session closed for user root 2024-09-20T22:13:53.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:53 smithi037 ceph-mon[23718]: pgmap v76: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-20T22:13:53.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:53 smithi037 ceph-mon[23718]: osdmap e22: 3 total, 3 up, 3 in 2024-09-20T22:13:53.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:53 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:13:54.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:53 smithi194 ceph-mon[25957]: pgmap v76: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-20T22:13:54.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:53 smithi194 ceph-mon[25957]: osdmap e22: 3 total, 3 up, 3 in 2024-09-20T22:13:54.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:53 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:13:54.186 INFO:journalctl@ceph.osd.1.smithi037.stdout:Sep 20 22:13:53 smithi037 sudo[42403]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-20T22:13:54.187 INFO:journalctl@ceph.osd.1.smithi037.stdout:Sep 20 22:13:53 smithi037 sudo[42403]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-20T22:13:54.187 INFO:journalctl@ceph.osd.1.smithi037.stdout:Sep 20 22:13:53 smithi037 sudo[42403]: pam_unix(sudo:session): session closed for user root 2024-09-20T22:13:54.437 INFO:journalctl@ceph.osd.2.smithi037.stdout:Sep 20 22:13:54 smithi037 sudo[42508]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-20T22:13:54.437 INFO:journalctl@ceph.osd.2.smithi037.stdout:Sep 20 22:13:54 smithi037 sudo[42508]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-20T22:13:54.438 INFO:journalctl@ceph.osd.2.smithi037.stdout:Sep 20 22:13:54 smithi037 sudo[42508]: pam_unix(sudo:session): session closed for user root 2024-09-20T22:13:55.140 INFO:journalctl@ceph.osd.2.smithi037.stdout:Sep 20 22:13:54 smithi037 sudo[42593]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-20T22:13:55.140 INFO:journalctl@ceph.osd.2.smithi037.stdout:Sep 20 22:13:54 smithi037 sudo[42593]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-20T22:13:55.140 INFO:journalctl@ceph.osd.2.smithi037.stdout:Sep 20 22:13:54 smithi037 sudo[42593]: pam_unix(sudo:session): session closed for user root 2024-09-20T22:13:55.412 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:55 smithi037 sudo[42598]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-20T22:13:55.412 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:55 smithi037 sudo[42598]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-20T22:13:55.412 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:55 smithi037 sudo[42598]: pam_unix(sudo:session): session closed for user root 2024-09-20T22:13:56.053 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:55 smithi194 ceph-mon[25957]: pgmap v78: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-20T22:13:56.053 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:55 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:13:56.053 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:55 smithi194 ceph-mon[25957]: from='client.14296 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi037:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:13:56.054 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:55 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-20T22:13:56.054 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:55 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-20T22:13:56.054 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:55 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:13:56.054 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:55 smithi194 ceph-mon[25957]: mgrmap e19: smithi037.lykblh(active, since 2m), standbys: smithi194.bkcglt 2024-09-20T22:13:56.054 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:55 smithi194 ceph-mon[25957]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-20T22:13:56.054 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:56 smithi194 sudo[28255]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-20T22:13:56.054 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:56 smithi194 sudo[28255]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-20T22:13:56.054 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:56 smithi194 sudo[28255]: pam_unix(sudo:session): session closed for user root 2024-09-20T22:13:56.105 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:55 smithi037 sudo[42630]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-20T22:13:56.105 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:55 smithi037 sudo[42630]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-20T22:13:56.105 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:55 smithi037 sudo[42630]: pam_unix(sudo:session): session closed for user root 2024-09-20T22:13:56.105 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:55 smithi037 ceph-mon[23718]: pgmap v78: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-20T22:13:56.105 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:55 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:13:56.105 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:55 smithi037 ceph-mon[23718]: from='client.14296 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi037:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:13:56.105 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:55 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-20T22:13:56.105 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:55 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-20T22:13:56.105 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:55 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:13:56.105 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:55 smithi037 ceph-mon[23718]: mgrmap e19: smithi037.lykblh(active, since 2m), standbys: smithi194.bkcglt 2024-09-20T22:13:56.106 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:55 smithi037 ceph-mon[23718]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-20T22:13:56.303 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:56 smithi194 sudo[28261]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-20T22:13:56.304 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:56 smithi194 sudo[28261]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-20T22:13:56.304 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:56 smithi194 sudo[28261]: pam_unix(sudo:session): session closed for user root 2024-09-20T22:13:57.044 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:56 smithi037 ceph-mon[23718]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-20T22:13:57.044 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:56 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mon metadata", "id": "smithi037"}]: dispatch 2024-09-20T22:13:57.044 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:56 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mon metadata", "id": "smithi194"}]: dispatch 2024-09-20T22:13:57.044 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:56 smithi037 ceph-mon[23718]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-20T22:13:57.044 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:56 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mon metadata", "id": "smithi037"}]: dispatch 2024-09-20T22:13:57.044 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:56 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mon metadata", "id": "smithi194"}]: dispatch 2024-09-20T22:13:57.044 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:56 smithi037 ceph-mon[23718]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-20T22:13:57.044 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:56 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:13:57.053 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:56 smithi194 ceph-mon[25957]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-20T22:13:57.053 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:56 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mon metadata", "id": "smithi037"}]: dispatch 2024-09-20T22:13:57.053 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:56 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mon metadata", "id": "smithi194"}]: dispatch 2024-09-20T22:13:57.053 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:56 smithi194 ceph-mon[25957]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-20T22:13:57.053 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:56 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mon metadata", "id": "smithi037"}]: dispatch 2024-09-20T22:13:57.053 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:56 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mon metadata", "id": "smithi194"}]: dispatch 2024-09-20T22:13:57.053 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:56 smithi194 ceph-mon[25957]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-20T22:13:57.053 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:56 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:13:58.078 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:57 smithi037 ceph-mon[23718]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2024-09-20T22:13:58.079 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:57 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:13:58.079 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:57 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-20T22:13:58.079 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:57 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-20T22:13:58.079 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:57 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-20T22:13:58.079 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:57 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:13:58.079 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:57 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:13:58.079 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:57 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:13:58.079 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:57 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:13:58.079 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:57 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:13:58.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:57 smithi194 ceph-mon[25957]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2024-09-20T22:13:58.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:57 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:13:58.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:57 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-20T22:13:58.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:57 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-20T22:13:58.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:57 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-20T22:13:58.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:57 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:13:58.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:57 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:13:58.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:57 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:13:58.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:57 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:13:58.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:57 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:13:58.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:58 smithi037 ceph-mon[23718]: Adjusting osd_memory_target on smithi037 to 4289M 2024-09-20T22:13:59.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:58 smithi194 ceph-mon[25957]: Adjusting osd_memory_target on smithi037 to 4289M 2024-09-20T22:14:00.079 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:59 smithi037 ceph-mon[23718]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-20T22:14:00.080 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:59 smithi037 ceph-mon[23718]: from='client.? 172.21.15.37:0/4061584754' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d555a027-d177-4ffa-9761-dd1a06b4c1a2"}]: dispatch 2024-09-20T22:14:00.080 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:59 smithi037 ceph-mon[23718]: from='client.? 172.21.15.37:0/4061584754' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d555a027-d177-4ffa-9761-dd1a06b4c1a2"}]': finished 2024-09-20T22:14:00.080 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:59 smithi037 ceph-mon[23718]: osdmap e23: 4 total, 3 up, 4 in 2024-09-20T22:14:00.080 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:59 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-20T22:14:00.080 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:13:59 smithi037 ceph-mon[23718]: from='client.? 172.21.15.37:0/2240203821' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-20T22:14:00.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:59 smithi194 ceph-mon[25957]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-20T22:14:00.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:59 smithi194 ceph-mon[25957]: from='client.? 172.21.15.37:0/4061584754' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d555a027-d177-4ffa-9761-dd1a06b4c1a2"}]: dispatch 2024-09-20T22:14:00.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:59 smithi194 ceph-mon[25957]: from='client.? 172.21.15.37:0/4061584754' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d555a027-d177-4ffa-9761-dd1a06b4c1a2"}]': finished 2024-09-20T22:14:00.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:59 smithi194 ceph-mon[25957]: osdmap e23: 4 total, 3 up, 4 in 2024-09-20T22:14:00.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:59 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-20T22:14:00.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:13:59 smithi194 ceph-mon[25957]: from='client.? 172.21.15.37:0/2240203821' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-20T22:14:02.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:01 smithi194 ceph-mon[25957]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 4.5 KiB/s rd, 196 KiB/s wr, 13 op/s 2024-09-20T22:14:02.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:01 smithi037 ceph-mon[23718]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 4.5 KiB/s rd, 196 KiB/s wr, 13 op/s 2024-09-20T22:14:03.739 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:03 smithi037 ceph-mon[23718]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-09-20T22:14:03.739 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:03 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:14:03.739 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:03 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:14:04.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:03 smithi194 ceph-mon[25957]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-09-20T22:14:04.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:03 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:14:04.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:03 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:14:05.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:05 smithi037 ceph-mon[23718]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-09-20T22:14:05.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:05 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-20T22:14:05.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:05 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:14:06.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:05 smithi194 ceph-mon[25957]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-09-20T22:14:06.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:05 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-20T22:14:06.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:05 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:14:06.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:06 smithi194 ceph-mon[25957]: Deploying daemon osd.3 on smithi037 2024-09-20T22:14:06.977 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:06 smithi037 ceph-mon[23718]: Deploying daemon osd.3 on smithi037 2024-09-20T22:14:07.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:07 smithi037 ceph-mon[23718]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-09-20T22:14:08.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:07 smithi194 ceph-mon[25957]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-09-20T22:14:09.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:09 smithi194 ceph-mon[25957]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-09-20T22:14:10.034 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:09 smithi037 ceph-mon[23718]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-09-20T22:14:11.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:11 smithi037 ceph-mon[23718]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.6 KiB/s rd, 70 KiB/s wr, 4 op/s 2024-09-20T22:14:11.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:11 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:14:11.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:11 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:14:11.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:11 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:14:11.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:11 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:14:12.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:11 smithi194 ceph-mon[25957]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.6 KiB/s rd, 70 KiB/s wr, 4 op/s 2024-09-20T22:14:12.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:11 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:14:12.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:11 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:14:12.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:11 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:14:12.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:11 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:14:14.030 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:13 smithi037 ceph-mon[23718]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-20T22:14:14.062 INFO:teuthology.orchestra.run.smithi037.stdout:Created osd(s) 3 on host 'smithi037' 2024-09-20T22:14:14.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:13 smithi194 ceph-mon[25957]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-20T22:14:14.660 DEBUG:teuthology.orchestra.run.smithi037:osd.3> sudo journalctl -f -n 0 -u ceph-f9a6357a-779c-11ef-baf5-efdc52797490@osd.3.service 2024-09-20T22:14:14.662 INFO:tasks.cephadm:Deploying osd.4 on smithi194 with /dev/vg_nvme/lv_4... 2024-09-20T22:14:14.663 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -- lvm zap /dev/vg_nvme/lv_4 2024-09-20T22:14:15.237 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:15 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:14:15.238 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:15 smithi194 ceph-mon[25957]: from='osd.3 [v2:172.21.15.37:6826/4193735334,v1:172.21.15.37:6827/4193735334]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-20T22:14:15.238 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:15 smithi194 ceph-mon[25957]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-20T22:14:15.412 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:15 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:14:15.412 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:15 smithi037 ceph-mon[23718]: from='osd.3 [v2:172.21.15.37:6826/4193735334,v1:172.21.15.37:6827/4193735334]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-20T22:14:15.412 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:15 smithi037 ceph-mon[23718]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-20T22:14:16.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:16 smithi194 ceph-mon[25957]: from='osd.3 [v2:172.21.15.37:6826/4193735334,v1:172.21.15.37:6827/4193735334]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-20T22:14:16.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:16 smithi194 ceph-mon[25957]: osdmap e24: 4 total, 3 up, 4 in 2024-09-20T22:14:16.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:16 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-20T22:14:16.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:16 smithi194 ceph-mon[25957]: from='osd.3 [v2:172.21.15.37:6826/4193735334,v1:172.21.15.37:6827/4193735334]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi037", "root=default"]}]: dispatch 2024-09-20T22:14:16.384 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:16 smithi037 ceph-mon[23718]: from='osd.3 [v2:172.21.15.37:6826/4193735334,v1:172.21.15.37:6827/4193735334]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-20T22:14:16.385 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:16 smithi037 ceph-mon[23718]: osdmap e24: 4 total, 3 up, 4 in 2024-09-20T22:14:16.385 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:16 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-20T22:14:16.385 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:16 smithi037 ceph-mon[23718]: from='osd.3 [v2:172.21.15.37:6826/4193735334,v1:172.21.15.37:6827/4193735334]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi037", "root=default"]}]: dispatch 2024-09-20T22:14:16.385 INFO:journalctl@ceph.osd.3.smithi037.stdout:Sep 20 22:14:16 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-3[44726]: 2024-09-20T22:14:16.070+0000 7fb5487b2700 -1 osd.3 0 waiting for initial osdmap 2024-09-20T22:14:16.385 INFO:journalctl@ceph.osd.3.smithi037.stdout:Sep 20 22:14:16 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-3[44726]: 2024-09-20T22:14:16.075+0000 7fb543149700 -1 osd.3 25 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-20T22:14:17.127 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-09-20T22:14:17.146 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -- ceph orch daemon add osd smithi194:vg_nvme/lv_4 2024-09-20T22:14:17.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:17 smithi194 ceph-mon[25957]: from='osd.3 [v2:172.21.15.37:6826/4193735334,v1:172.21.15.37:6827/4193735334]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi037", "root=default"]}]': finished 2024-09-20T22:14:17.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:17 smithi194 ceph-mon[25957]: osdmap e25: 4 total, 3 up, 4 in 2024-09-20T22:14:17.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:17 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-20T22:14:17.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:17 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-20T22:14:17.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:17 smithi194 ceph-mon[25957]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-20T22:14:17.412 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:17 smithi037 ceph-mon[23718]: from='osd.3 [v2:172.21.15.37:6826/4193735334,v1:172.21.15.37:6827/4193735334]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi037", "root=default"]}]': finished 2024-09-20T22:14:17.412 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:17 smithi037 ceph-mon[23718]: osdmap e25: 4 total, 3 up, 4 in 2024-09-20T22:14:17.412 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:17 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-20T22:14:17.412 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:17 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-20T22:14:17.412 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:17 smithi037 ceph-mon[23718]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-20T22:14:18.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:18 smithi194 ceph-mon[25957]: purged_snaps scrub starts 2024-09-20T22:14:18.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:18 smithi194 ceph-mon[25957]: purged_snaps scrub ok 2024-09-20T22:14:18.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:18 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-20T22:14:18.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:18 smithi194 ceph-mon[25957]: osd.3 [v2:172.21.15.37:6826/4193735334,v1:172.21.15.37:6827/4193735334] boot 2024-09-20T22:14:18.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:18 smithi194 ceph-mon[25957]: osdmap e26: 4 total, 4 up, 4 in 2024-09-20T22:14:18.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:18 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-20T22:14:18.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:18 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:14:18.408 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:18 smithi037 ceph-mon[23718]: purged_snaps scrub starts 2024-09-20T22:14:18.408 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:18 smithi037 ceph-mon[23718]: purged_snaps scrub ok 2024-09-20T22:14:18.408 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:18 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-20T22:14:18.408 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:18 smithi037 ceph-mon[23718]: osd.3 [v2:172.21.15.37:6826/4193735334,v1:172.21.15.37:6827/4193735334] boot 2024-09-20T22:14:18.408 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:18 smithi037 ceph-mon[23718]: osdmap e26: 4 total, 4 up, 4 in 2024-09-20T22:14:18.408 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:18 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-20T22:14:18.409 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:18 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:14:19.292 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:19 smithi037 ceph-mon[23718]: osdmap e27: 4 total, 4 up, 4 in 2024-09-20T22:14:19.293 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:19 smithi037 ceph-mon[23718]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T22:14:19.293 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:19 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-20T22:14:19.293 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:19 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-20T22:14:19.293 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:19 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:14:19.340 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:19 smithi194 ceph-mon[25957]: osdmap e27: 4 total, 4 up, 4 in 2024-09-20T22:14:19.340 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:19 smithi194 ceph-mon[25957]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T22:14:19.340 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:19 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-20T22:14:19.340 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:19 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-20T22:14:19.340 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:19 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:14:20.253 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:20 smithi194 ceph-mon[25957]: from='client.24153 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi194:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:14:20.412 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:20 smithi037 ceph-mon[23718]: from='client.24153 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi194:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:14:22.074 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:21 smithi037 ceph-mon[23718]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T22:14:22.074 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:21 smithi037 ceph-mon[23718]: Detected new or changed devices on smithi037 2024-09-20T22:14:22.074 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:21 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:14:22.074 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:21 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-20T22:14:22.074 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:21 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-20T22:14:22.074 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:21 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-20T22:14:22.074 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:21 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-20T22:14:22.074 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:21 smithi037 ceph-mon[23718]: Adjusting osd_memory_target on smithi037 to 3217M 2024-09-20T22:14:22.074 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:21 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:14:22.075 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:21 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:14:22.075 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:21 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:14:22.075 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:21 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:14:22.075 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:21 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:14:22.075 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:21 smithi037 ceph-mon[23718]: from='client.? 172.21.15.194:0/2067811211' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "268e1f18-01d9-4a1f-94b3-4cd5c6d0f843"}]: dispatch 2024-09-20T22:14:22.075 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:21 smithi037 ceph-mon[23718]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "268e1f18-01d9-4a1f-94b3-4cd5c6d0f843"}]: dispatch 2024-09-20T22:14:22.075 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:21 smithi037 ceph-mon[23718]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "268e1f18-01d9-4a1f-94b3-4cd5c6d0f843"}]': finished 2024-09-20T22:14:22.075 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:21 smithi037 ceph-mon[23718]: osdmap e28: 5 total, 4 up, 5 in 2024-09-20T22:14:22.075 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:21 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-20T22:14:22.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:21 smithi194 ceph-mon[25957]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T22:14:22.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:21 smithi194 ceph-mon[25957]: Detected new or changed devices on smithi037 2024-09-20T22:14:22.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:21 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:14:22.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:21 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-20T22:14:22.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:21 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-20T22:14:22.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:21 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-20T22:14:22.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:21 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-20T22:14:22.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:21 smithi194 ceph-mon[25957]: Adjusting osd_memory_target on smithi037 to 3217M 2024-09-20T22:14:22.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:21 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:14:22.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:21 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:14:22.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:21 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:14:22.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:21 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:14:22.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:21 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:14:22.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:21 smithi194 ceph-mon[25957]: from='client.? 172.21.15.194:0/2067811211' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "268e1f18-01d9-4a1f-94b3-4cd5c6d0f843"}]: dispatch 2024-09-20T22:14:22.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:21 smithi194 ceph-mon[25957]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "268e1f18-01d9-4a1f-94b3-4cd5c6d0f843"}]: dispatch 2024-09-20T22:14:22.096 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:21 smithi194 ceph-mon[25957]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "268e1f18-01d9-4a1f-94b3-4cd5c6d0f843"}]': finished 2024-09-20T22:14:22.096 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:21 smithi194 ceph-mon[25957]: osdmap e28: 5 total, 4 up, 5 in 2024-09-20T22:14:22.096 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:21 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-20T22:14:23.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:22 smithi194 ceph-mon[25957]: from='client.? 172.21.15.194:0/3102609902' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-20T22:14:23.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:22 smithi037 ceph-mon[23718]: from='client.? 172.21.15.194:0/3102609902' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-20T22:14:23.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:23 smithi037 ceph-mon[23718]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T22:14:24.042 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:23 smithi194 ceph-mon[25957]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T22:14:26.021 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:25 smithi194 ceph-mon[25957]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T22:14:26.021 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:25 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:14:26.021 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:25 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:14:26.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:25 smithi037 ceph-mon[23718]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T22:14:26.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:25 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:14:26.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:25 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:14:28.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:27 smithi194 ceph-mon[25957]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T22:14:28.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:27 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-20T22:14:28.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:27 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:14:28.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:27 smithi037 ceph-mon[23718]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T22:14:28.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:27 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-20T22:14:28.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:27 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:14:28.938 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:28 smithi194 ceph-mon[25957]: Deploying daemon osd.4 on smithi194 2024-09-20T22:14:29.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:28 smithi037 ceph-mon[23718]: Deploying daemon osd.4 on smithi194 2024-09-20T22:14:30.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:29 smithi194 ceph-mon[25957]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T22:14:30.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:29 smithi037 ceph-mon[23718]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T22:14:31.836 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:31 smithi194 ceph-mon[25957]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T22:14:32.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:31 smithi037 ceph-mon[23718]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T22:14:33.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:32 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:14:33.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:32 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:14:33.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:32 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:14:33.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:32 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:14:33.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:32 smithi037 ceph-mon[23718]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T22:14:33.193 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:32 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:14:33.194 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:32 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:14:33.194 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:32 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:14:33.194 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:32 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:14:33.194 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:32 smithi194 ceph-mon[25957]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T22:14:34.738 INFO:teuthology.orchestra.run.smithi194.stdout:Created osd(s) 4 on host 'smithi194' 2024-09-20T22:14:35.327 DEBUG:teuthology.orchestra.run.smithi194:osd.4> sudo journalctl -f -n 0 -u ceph-f9a6357a-779c-11ef-baf5-efdc52797490@osd.4.service 2024-09-20T22:14:35.330 INFO:tasks.cephadm:Deploying osd.5 on smithi194 with /dev/vg_nvme/lv_3... 2024-09-20T22:14:35.330 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -- lvm zap /dev/vg_nvme/lv_3 2024-09-20T22:14:35.828 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:35 smithi194 ceph-mon[25957]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T22:14:35.828 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:35 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:14:35.828 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:35 smithi194 ceph-mon[25957]: from='osd.4 [v2:172.21.15.194:6800/4219258866,v1:172.21.15.194:6801/4219258866]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-20T22:14:35.828 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:35 smithi194 ceph-mon[25957]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-20T22:14:36.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:35 smithi037 ceph-mon[23718]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T22:14:36.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:35 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:14:36.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:35 smithi037 ceph-mon[23718]: from='osd.4 [v2:172.21.15.194:6800/4219258866,v1:172.21.15.194:6801/4219258866]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-20T22:14:36.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:35 smithi037 ceph-mon[23718]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-20T22:14:37.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:36 smithi194 ceph-mon[25957]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-20T22:14:37.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:36 smithi194 ceph-mon[25957]: osdmap e29: 5 total, 4 up, 5 in 2024-09-20T22:14:37.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:36 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-20T22:14:37.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:36 smithi194 ceph-mon[25957]: from='osd.4 [v2:172.21.15.194:6800/4219258866,v1:172.21.15.194:6801/4219258866]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2024-09-20T22:14:37.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:36 smithi194 ceph-mon[25957]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2024-09-20T22:14:37.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:36 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:14:37.095 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 20 22:14:36 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-4[30172]: 2024-09-20T22:14:36.754+0000 7f0577e4d700 -1 osd.4 0 waiting for initial osdmap 2024-09-20T22:14:37.095 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 20 22:14:36 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-4[30172]: 2024-09-20T22:14:36.759+0000 7f0573fe7700 -1 osd.4 30 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-20T22:14:37.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:36 smithi037 ceph-mon[23718]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-20T22:14:37.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:36 smithi037 ceph-mon[23718]: osdmap e29: 5 total, 4 up, 5 in 2024-09-20T22:14:37.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:36 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-20T22:14:37.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:36 smithi037 ceph-mon[23718]: from='osd.4 [v2:172.21.15.194:6800/4219258866,v1:172.21.15.194:6801/4219258866]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2024-09-20T22:14:37.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:36 smithi037 ceph-mon[23718]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2024-09-20T22:14:37.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:36 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:14:38.003 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:37 smithi194 ceph-mon[25957]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T22:14:38.003 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:37 smithi194 ceph-mon[25957]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]': finished 2024-09-20T22:14:38.003 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:37 smithi194 ceph-mon[25957]: osdmap e30: 5 total, 4 up, 5 in 2024-09-20T22:14:38.003 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:37 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-20T22:14:38.003 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:37 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-20T22:14:38.003 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:37 smithi194 ceph-mon[25957]: osd.4 [v2:172.21.15.194:6800/4219258866,v1:172.21.15.194:6801/4219258866] boot 2024-09-20T22:14:38.004 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:37 smithi194 ceph-mon[25957]: osdmap e31: 5 total, 5 up, 5 in 2024-09-20T22:14:38.004 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:37 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-20T22:14:38.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:37 smithi037 ceph-mon[23718]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T22:14:38.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:37 smithi037 ceph-mon[23718]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]': finished 2024-09-20T22:14:38.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:37 smithi037 ceph-mon[23718]: osdmap e30: 5 total, 4 up, 5 in 2024-09-20T22:14:38.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:37 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-20T22:14:38.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:37 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-20T22:14:38.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:37 smithi037 ceph-mon[23718]: osd.4 [v2:172.21.15.194:6800/4219258866,v1:172.21.15.194:6801/4219258866] boot 2024-09-20T22:14:38.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:37 smithi037 ceph-mon[23718]: osdmap e31: 5 total, 5 up, 5 in 2024-09-20T22:14:38.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:37 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-20T22:14:39.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:38 smithi194 ceph-mon[25957]: purged_snaps scrub starts 2024-09-20T22:14:39.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:38 smithi194 ceph-mon[25957]: purged_snaps scrub ok 2024-09-20T22:14:39.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:38 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi037.lykblh/mirror_snapshot_schedule"}]: dispatch 2024-09-20T22:14:39.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:38 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi037.lykblh/trash_purge_schedule"}]: dispatch 2024-09-20T22:14:39.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:38 smithi194 ceph-mon[25957]: osdmap e32: 5 total, 5 up, 5 in 2024-09-20T22:14:39.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:38 smithi037 ceph-mon[23718]: purged_snaps scrub starts 2024-09-20T22:14:39.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:38 smithi037 ceph-mon[23718]: purged_snaps scrub ok 2024-09-20T22:14:39.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:38 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi037.lykblh/mirror_snapshot_schedule"}]: dispatch 2024-09-20T22:14:39.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:38 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi037.lykblh/trash_purge_schedule"}]: dispatch 2024-09-20T22:14:39.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:38 smithi037 ceph-mon[23718]: osdmap e32: 5 total, 5 up, 5 in 2024-09-20T22:14:39.407 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-09-20T22:14:39.427 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -- ceph orch daemon add osd smithi194:vg_nvme/lv_3 2024-09-20T22:14:40.015 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:39 smithi194 ceph-mon[25957]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-09-20T22:14:40.016 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:39 smithi194 ceph-mon[25957]: osdmap e33: 5 total, 5 up, 5 in 2024-09-20T22:14:40.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:39 smithi037 ceph-mon[23718]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-09-20T22:14:40.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:39 smithi037 ceph-mon[23718]: osdmap e33: 5 total, 5 up, 5 in 2024-09-20T22:14:41.801 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:41 smithi194 ceph-mon[25957]: pgmap v112: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-09-20T22:14:41.801 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:41 smithi194 ceph-mon[25957]: osdmap e34: 5 total, 5 up, 5 in 2024-09-20T22:14:41.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:41 smithi037 ceph-mon[23718]: pgmap v112: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-09-20T22:14:41.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:41 smithi037 ceph-mon[23718]: osdmap e34: 5 total, 5 up, 5 in 2024-09-20T22:14:42.974 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:42 smithi194 ceph-mon[25957]: osdmap e35: 5 total, 5 up, 5 in 2024-09-20T22:14:42.974 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:42 smithi194 ceph-mon[25957]: from='client.24171 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi194:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:14:42.974 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:42 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-20T22:14:42.975 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:42 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-20T22:14:42.975 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:42 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:14:42.975 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:42 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:14:42.975 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:42 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-20T22:14:42.975 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:42 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:14:42.975 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:42 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:14:42.975 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:42 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:14:42.975 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:42 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:14:42.975 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:42 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:14:42.976 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:42 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:14:43.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:42 smithi037 ceph-mon[23718]: osdmap e35: 5 total, 5 up, 5 in 2024-09-20T22:14:43.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:42 smithi037 ceph-mon[23718]: from='client.24171 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi194:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:14:43.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:42 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-20T22:14:43.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:42 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-20T22:14:43.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:42 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:14:43.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:42 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:14:43.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:42 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-20T22:14:43.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:42 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:14:43.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:42 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:14:43.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:42 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:14:43.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:42 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:14:43.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:42 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:14:43.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:42 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:14:44.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:43 smithi194 ceph-mon[25957]: Detected new or changed devices on smithi194 2024-09-20T22:14:44.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:43 smithi194 ceph-mon[25957]: Adjusting osd_memory_target on smithi194 to 15936M 2024-09-20T22:14:44.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:43 smithi194 ceph-mon[25957]: pgmap v115: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-09-20T22:14:44.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:43 smithi037 ceph-mon[23718]: Detected new or changed devices on smithi194 2024-09-20T22:14:44.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:43 smithi037 ceph-mon[23718]: Adjusting osd_memory_target on smithi194 to 15936M 2024-09-20T22:14:44.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:43 smithi037 ceph-mon[23718]: pgmap v115: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-09-20T22:14:45.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:44 smithi194 ceph-mon[25957]: from='client.? 172.21.15.194:0/2738145291' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "06dd9a31-ccb8-4238-a4d8-ca6edc0ec774"}]: dispatch 2024-09-20T22:14:45.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:44 smithi194 ceph-mon[25957]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "06dd9a31-ccb8-4238-a4d8-ca6edc0ec774"}]: dispatch 2024-09-20T22:14:45.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:44 smithi194 ceph-mon[25957]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "06dd9a31-ccb8-4238-a4d8-ca6edc0ec774"}]': finished 2024-09-20T22:14:45.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:44 smithi194 ceph-mon[25957]: osdmap e36: 6 total, 5 up, 6 in 2024-09-20T22:14:45.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:44 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-20T22:14:45.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:44 smithi194 ceph-mon[25957]: from='client.? 172.21.15.194:0/3290224285' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-20T22:14:45.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:44 smithi037 ceph-mon[23718]: from='client.? 172.21.15.194:0/2738145291' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "06dd9a31-ccb8-4238-a4d8-ca6edc0ec774"}]: dispatch 2024-09-20T22:14:45.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:44 smithi037 ceph-mon[23718]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "06dd9a31-ccb8-4238-a4d8-ca6edc0ec774"}]: dispatch 2024-09-20T22:14:45.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:44 smithi037 ceph-mon[23718]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "06dd9a31-ccb8-4238-a4d8-ca6edc0ec774"}]': finished 2024-09-20T22:14:45.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:44 smithi037 ceph-mon[23718]: osdmap e36: 6 total, 5 up, 6 in 2024-09-20T22:14:45.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:44 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-20T22:14:45.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:44 smithi037 ceph-mon[23718]: from='client.? 172.21.15.194:0/3290224285' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-20T22:14:46.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:45 smithi194 ceph-mon[25957]: pgmap v117: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 115 KiB/s, 0 objects/s recovering 2024-09-20T22:14:46.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:45 smithi037 ceph-mon[23718]: pgmap v117: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 115 KiB/s, 0 objects/s recovering 2024-09-20T22:14:47.875 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:47 smithi194 ceph-mon[25957]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-20T22:14:48.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:47 smithi037 ceph-mon[23718]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-20T22:14:50.025 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:49 smithi194 ceph-mon[25957]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-20T22:14:50.025 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:49 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-20T22:14:50.025 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:49 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:14:50.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:49 smithi037 ceph-mon[23718]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-20T22:14:50.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:49 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-20T22:14:50.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:49 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:14:51.031 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:50 smithi194 ceph-mon[25957]: Deploying daemon osd.5 on smithi194 2024-09-20T22:14:51.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:50 smithi037 ceph-mon[23718]: Deploying daemon osd.5 on smithi194 2024-09-20T22:14:51.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:51 smithi037 ceph-mon[23718]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-20T22:14:52.031 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:51 smithi194 ceph-mon[25957]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-20T22:14:53.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:53 smithi194 ceph-mon[25957]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-20T22:14:54.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:53 smithi037 ceph-mon[23718]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-20T22:14:55.703 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:55 smithi194 ceph-mon[25957]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-20T22:14:55.704 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:55 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:14:55.704 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:55 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:14:55.704 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:55 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:14:55.704 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:55 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:14:55.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:55 smithi037 ceph-mon[23718]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-20T22:14:55.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:55 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:14:55.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:55 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:14:55.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:55 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:14:55.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:55 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:14:57.840 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:57 smithi194 ceph-mon[25957]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-20T22:14:57.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:57 smithi037 ceph-mon[23718]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-20T22:14:58.122 INFO:teuthology.orchestra.run.smithi194.stdout:Created osd(s) 5 on host 'smithi194' 2024-09-20T22:14:58.694 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:58 smithi194 ceph-mon[25957]: from='osd.5 [v2:172.21.15.194:6808/3326488387,v1:172.21.15.194:6809/3326488387]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-20T22:14:58.694 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:58 smithi194 ceph-mon[25957]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-20T22:14:58.694 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:58 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:14:58.717 DEBUG:teuthology.orchestra.run.smithi194:osd.5> sudo journalctl -f -n 0 -u ceph-f9a6357a-779c-11ef-baf5-efdc52797490@osd.5.service 2024-09-20T22:14:58.719 INFO:tasks.cephadm:Deploying osd.6 on smithi194 with /dev/vg_nvme/lv_2... 2024-09-20T22:14:58.720 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -- lvm zap /dev/vg_nvme/lv_2 2024-09-20T22:14:58.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:58 smithi037 ceph-mon[23718]: from='osd.5 [v2:172.21.15.194:6808/3326488387,v1:172.21.15.194:6809/3326488387]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-20T22:14:58.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:58 smithi037 ceph-mon[23718]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-20T22:14:58.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:58 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:14:59.699 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:59 smithi194 ceph-mon[25957]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-20T22:14:59.700 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:59 smithi194 ceph-mon[25957]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-20T22:14:59.700 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:59 smithi194 ceph-mon[25957]: osdmap e37: 6 total, 5 up, 6 in 2024-09-20T22:14:59.700 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:59 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-20T22:14:59.700 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:59 smithi194 ceph-mon[25957]: from='osd.5 [v2:172.21.15.194:6808/3326488387,v1:172.21.15.194:6809/3326488387]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2024-09-20T22:14:59.700 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:59 smithi194 ceph-mon[25957]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2024-09-20T22:14:59.700 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:14:59 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:14:59.700 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:14:59 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-5[33124]: 2024-09-20T22:14:59.455+0000 7fb47146e700 -1 osd.5 0 waiting for initial osdmap 2024-09-20T22:14:59.701 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:14:59 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-5[33124]: 2024-09-20T22:14:59.460+0000 7fb46b604700 -1 osd.5 38 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-20T22:14:59.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:59 smithi037 ceph-mon[23718]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-20T22:14:59.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:59 smithi037 ceph-mon[23718]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-20T22:14:59.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:59 smithi037 ceph-mon[23718]: osdmap e37: 6 total, 5 up, 6 in 2024-09-20T22:14:59.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:59 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-20T22:14:59.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:59 smithi037 ceph-mon[23718]: from='osd.5 [v2:172.21.15.194:6808/3326488387,v1:172.21.15.194:6809/3326488387]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2024-09-20T22:14:59.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:59 smithi037 ceph-mon[23718]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2024-09-20T22:14:59.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:14:59 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:15:00.654 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:00 smithi194 ceph-mon[25957]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]': finished 2024-09-20T22:15:00.654 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:00 smithi194 ceph-mon[25957]: osdmap e38: 6 total, 5 up, 6 in 2024-09-20T22:15:00.654 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:00 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-20T22:15:00.654 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:00 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-20T22:15:00.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:00 smithi037 ceph-mon[23718]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]': finished 2024-09-20T22:15:00.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:00 smithi037 ceph-mon[23718]: osdmap e38: 6 total, 5 up, 6 in 2024-09-20T22:15:00.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:00 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-20T22:15:00.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:00 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-20T22:15:01.652 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:01 smithi194 ceph-mon[25957]: purged_snaps scrub starts 2024-09-20T22:15:01.652 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:01 smithi194 ceph-mon[25957]: purged_snaps scrub ok 2024-09-20T22:15:01.652 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:01 smithi194 ceph-mon[25957]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-20T22:15:01.652 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:01 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-20T22:15:01.652 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:01 smithi194 ceph-mon[25957]: osd.5 [v2:172.21.15.194:6808/3326488387,v1:172.21.15.194:6809/3326488387] boot 2024-09-20T22:15:01.652 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:01 smithi194 ceph-mon[25957]: osdmap e39: 6 total, 6 up, 6 in 2024-09-20T22:15:01.652 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:01 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-20T22:15:01.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:01 smithi037 ceph-mon[23718]: purged_snaps scrub starts 2024-09-20T22:15:01.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:01 smithi037 ceph-mon[23718]: purged_snaps scrub ok 2024-09-20T22:15:01.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:01 smithi037 ceph-mon[23718]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-20T22:15:01.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:01 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-20T22:15:01.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:01 smithi037 ceph-mon[23718]: osd.5 [v2:172.21.15.194:6808/3326488387,v1:172.21.15.194:6809/3326488387] boot 2024-09-20T22:15:01.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:01 smithi037 ceph-mon[23718]: osdmap e39: 6 total, 6 up, 6 in 2024-09-20T22:15:01.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:01 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-20T22:15:02.130 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-09-20T22:15:02.154 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -- ceph orch daemon add osd smithi194:vg_nvme/lv_2 2024-09-20T22:15:02.751 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:02 smithi194 ceph-mon[25957]: osdmap e40: 6 total, 6 up, 6 in 2024-09-20T22:15:02.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:02 smithi037 ceph-mon[23718]: osdmap e40: 6 total, 6 up, 6 in 2024-09-20T22:15:03.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:03 smithi194 ceph-mon[25957]: pgmap v130: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-20T22:15:03.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:03 smithi194 ceph-mon[25957]: osdmap e41: 6 total, 6 up, 6 in 2024-09-20T22:15:03.813 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:03 smithi037 ceph-mon[23718]: pgmap v130: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-20T22:15:03.814 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:03 smithi037 ceph-mon[23718]: osdmap e41: 6 total, 6 up, 6 in 2024-09-20T22:15:04.571 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:04 smithi194 ceph-mon[25957]: osdmap e42: 6 total, 6 up, 6 in 2024-09-20T22:15:04.571 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:04 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:15:04.571 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:04 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-20T22:15:04.572 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:04 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-20T22:15:04.572 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:04 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:15:04.572 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:04 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:15:04.572 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:04 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:15:04.572 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:04 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:15:04.572 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:04 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:15:04.572 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:04 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:15:04.572 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:04 smithi194 ceph-mon[25957]: osdmap e43: 6 total, 6 up, 6 in 2024-09-20T22:15:04.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:04 smithi037 ceph-mon[23718]: osdmap e42: 6 total, 6 up, 6 in 2024-09-20T22:15:04.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:04 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:15:04.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:04 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-20T22:15:04.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:04 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-20T22:15:04.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:04 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:15:04.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:04 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:15:04.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:04 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:15:04.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:04 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:15:04.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:04 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:15:04.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:04 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:15:04.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:04 smithi037 ceph-mon[23718]: osdmap e43: 6 total, 6 up, 6 in 2024-09-20T22:15:05.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:05 smithi194 ceph-mon[25957]: Detected new or changed devices on smithi194 2024-09-20T22:15:05.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:05 smithi194 ceph-mon[25957]: Adjusting osd_memory_target on smithi194 to 7968M 2024-09-20T22:15:05.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:05 smithi194 ceph-mon[25957]: pgmap v133: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-20T22:15:05.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:05 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-20T22:15:05.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:05 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-20T22:15:05.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:05 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:15:05.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:05 smithi037 ceph-mon[23718]: Detected new or changed devices on smithi194 2024-09-20T22:15:05.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:05 smithi037 ceph-mon[23718]: Adjusting osd_memory_target on smithi194 to 7968M 2024-09-20T22:15:05.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:05 smithi037 ceph-mon[23718]: pgmap v133: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-20T22:15:05.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:05 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-20T22:15:05.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:05 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-20T22:15:05.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:05 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:15:06.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:06 smithi194 ceph-mon[25957]: from='client.24189 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi194:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:15:06.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:06 smithi037 ceph-mon[23718]: from='client.24189 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi194:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:15:07.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:07 smithi194 ceph-mon[25957]: pgmap v135: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-20T22:15:07.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:07 smithi194 ceph-mon[25957]: from='client.? 172.21.15.194:0/1913670860' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2ae4c518-af42-4827-bb85-8e8a11b72ea2"}]: dispatch 2024-09-20T22:15:07.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:07 smithi194 ceph-mon[25957]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2ae4c518-af42-4827-bb85-8e8a11b72ea2"}]: dispatch 2024-09-20T22:15:07.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:07 smithi194 ceph-mon[25957]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "2ae4c518-af42-4827-bb85-8e8a11b72ea2"}]': finished 2024-09-20T22:15:07.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:07 smithi194 ceph-mon[25957]: osdmap e44: 7 total, 6 up, 7 in 2024-09-20T22:15:07.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:07 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-20T22:15:07.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:07 smithi037 ceph-mon[23718]: pgmap v135: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-20T22:15:07.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:07 smithi037 ceph-mon[23718]: from='client.? 172.21.15.194:0/1913670860' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2ae4c518-af42-4827-bb85-8e8a11b72ea2"}]: dispatch 2024-09-20T22:15:07.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:07 smithi037 ceph-mon[23718]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2ae4c518-af42-4827-bb85-8e8a11b72ea2"}]: dispatch 2024-09-20T22:15:07.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:07 smithi037 ceph-mon[23718]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "2ae4c518-af42-4827-bb85-8e8a11b72ea2"}]': finished 2024-09-20T22:15:07.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:07 smithi037 ceph-mon[23718]: osdmap e44: 7 total, 6 up, 7 in 2024-09-20T22:15:07.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:07 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-20T22:15:08.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:08 smithi194 ceph-mon[25957]: from='client.? 172.21.15.194:0/3043411675' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-20T22:15:08.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:08 smithi037 ceph-mon[23718]: from='client.? 172.21.15.194:0/3043411675' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-20T22:15:09.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:09 smithi194 ceph-mon[25957]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-20T22:15:10.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:09 smithi037 ceph-mon[23718]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-20T22:15:11.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:11 smithi037 ceph-mon[23718]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-20T22:15:12.008 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:11 smithi194 ceph-mon[25957]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-20T22:15:13.982 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:13 smithi194 ceph-mon[25957]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-20T22:15:13.982 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:13 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-20T22:15:13.982 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:13 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:15:14.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:13 smithi037 ceph-mon[23718]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-20T22:15:14.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:13 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-20T22:15:14.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:13 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:15:14.810 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:14 smithi194 ceph-mon[25957]: Deploying daemon osd.6 on smithi194 2024-09-20T22:15:15.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:14 smithi037 ceph-mon[23718]: Deploying daemon osd.6 on smithi194 2024-09-20T22:15:15.929 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:15 smithi194 ceph-mon[25957]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-20T22:15:16.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:15 smithi037 ceph-mon[23718]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-20T22:15:17.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:17 smithi194 ceph-mon[25957]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-20T22:15:18.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:17 smithi037 ceph-mon[23718]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-20T22:15:19.438 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:19 smithi194 ceph-mon[25957]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-20T22:15:19.438 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:19 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:15:19.438 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:19 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:15:19.438 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:19 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:15:19.438 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:19 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:15:19.662 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:19 smithi037 ceph-mon[23718]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-20T22:15:19.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:19 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:15:19.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:19 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:15:19.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:19 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:15:19.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:19 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:15:21.290 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:21 smithi194 ceph-mon[25957]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-20T22:15:21.336 INFO:teuthology.orchestra.run.smithi194.stdout:Created osd(s) 6 on host 'smithi194' 2024-09-20T22:15:21.503 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:21 smithi037 ceph-mon[23718]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-20T22:15:21.941 DEBUG:teuthology.orchestra.run.smithi194:osd.6> sudo journalctl -f -n 0 -u ceph-f9a6357a-779c-11ef-baf5-efdc52797490@osd.6.service 2024-09-20T22:15:21.943 INFO:tasks.cephadm:Deploying osd.7 on smithi194 with /dev/vg_nvme/lv_1... 2024-09-20T22:15:21.943 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -- lvm zap /dev/vg_nvme/lv_1 2024-09-20T22:15:22.205 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:22 smithi194 ceph-mon[25957]: from='osd.6 [v2:172.21.15.194:6816/1380757103,v1:172.21.15.194:6817/1380757103]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-20T22:15:22.205 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:22 smithi194 ceph-mon[25957]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-20T22:15:22.205 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:22 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:15:22.662 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:22 smithi037 ceph-mon[23718]: from='osd.6 [v2:172.21.15.194:6816/1380757103,v1:172.21.15.194:6817/1380757103]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-20T22:15:22.662 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:22 smithi037 ceph-mon[23718]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-20T22:15:22.662 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:22 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:15:23.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:23 smithi194 ceph-mon[25957]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-20T22:15:23.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:23 smithi194 ceph-mon[25957]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-20T22:15:23.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:23 smithi194 ceph-mon[25957]: osdmap e45: 7 total, 6 up, 7 in 2024-09-20T22:15:23.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:23 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-20T22:15:23.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:23 smithi194 ceph-mon[25957]: from='osd.6 [v2:172.21.15.194:6816/1380757103,v1:172.21.15.194:6817/1380757103]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2024-09-20T22:15:23.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:23 smithi194 ceph-mon[25957]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2024-09-20T22:15:23.345 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:15:23 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6[36079]: 2024-09-20T22:15:23.187+0000 7f68d93f2700 -1 osd.6 0 waiting for initial osdmap 2024-09-20T22:15:23.345 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:15:23 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6[36079]: 2024-09-20T22:15:23.194+0000 7f68d558c700 -1 osd.6 46 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-20T22:15:23.412 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:23 smithi037 ceph-mon[23718]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-20T22:15:23.412 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:23 smithi037 ceph-mon[23718]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-20T22:15:23.412 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:23 smithi037 ceph-mon[23718]: osdmap e45: 7 total, 6 up, 7 in 2024-09-20T22:15:23.412 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:23 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-20T22:15:23.412 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:23 smithi037 ceph-mon[23718]: from='osd.6 [v2:172.21.15.194:6816/1380757103,v1:172.21.15.194:6817/1380757103]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2024-09-20T22:15:23.412 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:23 smithi037 ceph-mon[23718]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2024-09-20T22:15:24.195 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:24 smithi194 ceph-mon[25957]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]': finished 2024-09-20T22:15:24.196 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:24 smithi194 ceph-mon[25957]: osdmap e46: 7 total, 6 up, 7 in 2024-09-20T22:15:24.196 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:24 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-20T22:15:24.196 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:24 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-20T22:15:24.196 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:24 smithi194 ceph-mon[25957]: osd.6 [v2:172.21.15.194:6816/1380757103,v1:172.21.15.194:6817/1380757103] boot 2024-09-20T22:15:24.196 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:24 smithi194 ceph-mon[25957]: osdmap e47: 7 total, 7 up, 7 in 2024-09-20T22:15:24.196 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:24 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-20T22:15:24.662 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:24 smithi037 ceph-mon[23718]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]': finished 2024-09-20T22:15:24.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:24 smithi037 ceph-mon[23718]: osdmap e46: 7 total, 6 up, 7 in 2024-09-20T22:15:24.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:24 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-20T22:15:24.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:24 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-20T22:15:24.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:24 smithi037 ceph-mon[23718]: osd.6 [v2:172.21.15.194:6816/1380757103,v1:172.21.15.194:6817/1380757103] boot 2024-09-20T22:15:24.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:24 smithi037 ceph-mon[23718]: osdmap e47: 7 total, 7 up, 7 in 2024-09-20T22:15:24.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:24 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-20T22:15:25.090 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-09-20T22:15:25.109 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -- ceph orch daemon add osd smithi194:vg_nvme/lv_1 2024-09-20T22:15:25.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:25 smithi194 ceph-mon[25957]: purged_snaps scrub starts 2024-09-20T22:15:25.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:25 smithi194 ceph-mon[25957]: purged_snaps scrub ok 2024-09-20T22:15:25.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:25 smithi194 ceph-mon[25957]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-20T22:15:25.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:25 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:15:25.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:25 smithi194 ceph-mon[25957]: osdmap e48: 7 total, 7 up, 7 in 2024-09-20T22:15:25.662 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:25 smithi037 ceph-mon[23718]: purged_snaps scrub starts 2024-09-20T22:15:25.662 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:25 smithi037 ceph-mon[23718]: purged_snaps scrub ok 2024-09-20T22:15:25.662 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:25 smithi037 ceph-mon[23718]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-20T22:15:25.662 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:25 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:15:25.662 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:25 smithi037 ceph-mon[23718]: osdmap e48: 7 total, 7 up, 7 in 2024-09-20T22:15:26.763 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:26 smithi194 ceph-mon[25957]: osdmap e49: 7 total, 7 up, 7 in 2024-09-20T22:15:27.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:26 smithi037 ceph-mon[23718]: osdmap e49: 7 total, 7 up, 7 in 2024-09-20T22:15:27.907 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:27 smithi194 ceph-mon[25957]: pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-09-20T22:15:27.907 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:27 smithi194 ceph-mon[25957]: osdmap e50: 7 total, 7 up, 7 in 2024-09-20T22:15:28.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:27 smithi037 ceph-mon[23718]: pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-09-20T22:15:28.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:27 smithi037 ceph-mon[23718]: osdmap e50: 7 total, 7 up, 7 in 2024-09-20T22:15:28.797 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:28 smithi194 ceph-mon[25957]: osdmap e51: 7 total, 7 up, 7 in 2024-09-20T22:15:28.797 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:28 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-20T22:15:28.797 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:28 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-20T22:15:28.797 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:28 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:15:29.059 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:28 smithi037 ceph-mon[23718]: osdmap e51: 7 total, 7 up, 7 in 2024-09-20T22:15:29.059 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:28 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-20T22:15:29.059 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:28 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-20T22:15:29.060 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:28 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:15:30.005 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:29 smithi194 ceph-mon[25957]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-20T22:15:30.005 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:29 smithi194 ceph-mon[25957]: from='client.24207 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi194:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:15:30.006 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:29 smithi194 ceph-mon[25957]: Detected new or changed devices on smithi194 2024-09-20T22:15:30.006 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:29 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:15:30.006 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:29 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-20T22:15:30.006 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:29 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-20T22:15:30.006 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:29 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-20T22:15:30.006 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:29 smithi194 ceph-mon[25957]: Adjusting osd_memory_target on smithi194 to 5312M 2024-09-20T22:15:30.006 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:29 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:15:30.006 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:29 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:15:30.006 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:29 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:15:30.006 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:29 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:15:30.006 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:29 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:15:30.006 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:29 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:15:30.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:29 smithi037 ceph-mon[23718]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-20T22:15:30.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:29 smithi037 ceph-mon[23718]: from='client.24207 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi194:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:15:30.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:29 smithi037 ceph-mon[23718]: Detected new or changed devices on smithi194 2024-09-20T22:15:30.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:29 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:15:30.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:29 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-20T22:15:30.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:29 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-20T22:15:30.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:29 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-20T22:15:30.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:29 smithi037 ceph-mon[23718]: Adjusting osd_memory_target on smithi194 to 5312M 2024-09-20T22:15:30.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:29 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:15:30.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:29 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:15:30.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:29 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:15:30.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:29 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:15:30.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:29 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:15:30.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:29 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:15:31.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:31 smithi037 ceph-mon[23718]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 94 KiB/s, 0 objects/s recovering 2024-09-20T22:15:31.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:31 smithi037 ceph-mon[23718]: from='client.? 172.21.15.194:0/2879063008' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8acd1795-badf-4632-99b1-f61a81bce2cf"}]: dispatch 2024-09-20T22:15:31.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:31 smithi037 ceph-mon[23718]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8acd1795-badf-4632-99b1-f61a81bce2cf"}]: dispatch 2024-09-20T22:15:31.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:31 smithi037 ceph-mon[23718]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "8acd1795-badf-4632-99b1-f61a81bce2cf"}]': finished 2024-09-20T22:15:31.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:31 smithi037 ceph-mon[23718]: osdmap e52: 8 total, 7 up, 8 in 2024-09-20T22:15:31.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:31 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-20T22:15:31.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:31 smithi037 ceph-mon[23718]: from='client.? 172.21.15.194:0/2738955282' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-20T22:15:32.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:31 smithi194 ceph-mon[25957]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 94 KiB/s, 0 objects/s recovering 2024-09-20T22:15:32.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:31 smithi194 ceph-mon[25957]: from='client.? 172.21.15.194:0/2879063008' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8acd1795-badf-4632-99b1-f61a81bce2cf"}]: dispatch 2024-09-20T22:15:32.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:31 smithi194 ceph-mon[25957]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8acd1795-badf-4632-99b1-f61a81bce2cf"}]: dispatch 2024-09-20T22:15:32.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:31 smithi194 ceph-mon[25957]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "8acd1795-badf-4632-99b1-f61a81bce2cf"}]': finished 2024-09-20T22:15:32.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:31 smithi194 ceph-mon[25957]: osdmap e52: 8 total, 7 up, 8 in 2024-09-20T22:15:32.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:31 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-20T22:15:32.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:31 smithi194 ceph-mon[25957]: from='client.? 172.21.15.194:0/2738955282' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-20T22:15:33.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:33 smithi194 ceph-mon[25957]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-20T22:15:34.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:33 smithi037 ceph-mon[23718]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-20T22:15:35.980 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:35 smithi194 ceph-mon[25957]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 69 KiB/s, 0 objects/s recovering 2024-09-20T22:15:36.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:35 smithi037 ceph-mon[23718]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 69 KiB/s, 0 objects/s recovering 2024-09-20T22:15:38.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:37 smithi194 ceph-mon[25957]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 61 KiB/s, 0 objects/s recovering 2024-09-20T22:15:38.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:37 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-20T22:15:38.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:37 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:15:38.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:37 smithi037 ceph-mon[23718]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 61 KiB/s, 0 objects/s recovering 2024-09-20T22:15:38.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:37 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-20T22:15:38.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:37 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:15:39.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:38 smithi194 ceph-mon[25957]: Deploying daemon osd.7 on smithi194 2024-09-20T22:15:39.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:38 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi037.lykblh/mirror_snapshot_schedule"}]: dispatch 2024-09-20T22:15:39.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:38 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi037.lykblh/trash_purge_schedule"}]: dispatch 2024-09-20T22:15:39.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:38 smithi037 ceph-mon[23718]: Deploying daemon osd.7 on smithi194 2024-09-20T22:15:39.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:38 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi037.lykblh/mirror_snapshot_schedule"}]: dispatch 2024-09-20T22:15:39.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:38 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi037.lykblh/trash_purge_schedule"}]: dispatch 2024-09-20T22:15:40.065 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:39 smithi194 ceph-mon[25957]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-20T22:15:40.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:39 smithi037 ceph-mon[23718]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-20T22:15:41.841 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:41 smithi194 ceph-mon[25957]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-20T22:15:42.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:41 smithi037 ceph-mon[23718]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-20T22:15:43.297 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:43 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:15:43.297 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:43 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:15:43.297 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:43 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:15:43.297 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:43 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:15:43.297 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:43 smithi194 ceph-mon[25957]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-20T22:15:43.412 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:43 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:15:43.412 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:43 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:15:43.412 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:43 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:15:43.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:43 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:15:43.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:43 smithi037 ceph-mon[23718]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-20T22:15:45.373 INFO:teuthology.orchestra.run.smithi194.stdout:Created osd(s) 7 on host 'smithi194' 2024-09-20T22:15:45.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:45 smithi037 ceph-mon[23718]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-20T22:15:45.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:45 smithi037 ceph-mon[23718]: from='osd.7 [v2:172.21.15.194:6824/2294923533,v1:172.21.15.194:6825/2294923533]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-20T22:15:45.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:45 smithi037 ceph-mon[23718]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-20T22:15:45.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:45 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:15:45.961 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:45 smithi194 ceph-mon[25957]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-20T22:15:45.961 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:45 smithi194 ceph-mon[25957]: from='osd.7 [v2:172.21.15.194:6824/2294923533,v1:172.21.15.194:6825/2294923533]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-20T22:15:45.961 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:45 smithi194 ceph-mon[25957]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-20T22:15:45.961 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:45 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:15:45.985 DEBUG:teuthology.orchestra.run.smithi194:osd.7> sudo journalctl -f -n 0 -u ceph-f9a6357a-779c-11ef-baf5-efdc52797490@osd.7.service 2024-09-20T22:15:45.987 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-09-20T22:15:45.988 DEBUG:teuthology.orchestra.run.smithi037:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -- ceph osd stat -f json 2024-09-20T22:15:46.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:46 smithi194 ceph-mon[25957]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-20T22:15:46.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:46 smithi194 ceph-mon[25957]: osdmap e53: 8 total, 7 up, 8 in 2024-09-20T22:15:46.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:46 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-20T22:15:46.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:46 smithi194 ceph-mon[25957]: from='osd.7 [v2:172.21.15.194:6824/2294923533,v1:172.21.15.194:6825/2294923533]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2024-09-20T22:15:46.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:46 smithi194 ceph-mon[25957]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2024-09-20T22:15:46.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:46 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:15:46.845 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:15:46 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[39126]: 2024-09-20T22:15:46.693+0000 7f6d52bd7700 -1 osd.7 0 waiting for initial osdmap 2024-09-20T22:15:46.845 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:15:46 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[39126]: 2024-09-20T22:15:46.701+0000 7f6d4dd6f700 -1 osd.7 54 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-20T22:15:46.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:46 smithi037 ceph-mon[23718]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-20T22:15:46.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:46 smithi037 ceph-mon[23718]: osdmap e53: 8 total, 7 up, 8 in 2024-09-20T22:15:46.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:46 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-20T22:15:46.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:46 smithi037 ceph-mon[23718]: from='osd.7 [v2:172.21.15.194:6824/2294923533,v1:172.21.15.194:6825/2294923533]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2024-09-20T22:15:46.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:46 smithi037 ceph-mon[23718]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2024-09-20T22:15:46.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:46 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:15:47.967 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:47 smithi194 ceph-mon[25957]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-20T22:15:47.967 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:47 smithi194 ceph-mon[25957]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]': finished 2024-09-20T22:15:47.967 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:47 smithi194 ceph-mon[25957]: osdmap e54: 8 total, 7 up, 8 in 2024-09-20T22:15:47.967 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:47 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-20T22:15:47.967 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:47 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-20T22:15:48.086 INFO:teuthology.orchestra.run.smithi037.stdout: 2024-09-20T22:15:48.086 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:47 smithi037 ceph-mon[23718]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-20T22:15:48.086 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:47 smithi037 ceph-mon[23718]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]': finished 2024-09-20T22:15:48.086 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:47 smithi037 ceph-mon[23718]: osdmap e54: 8 total, 7 up, 8 in 2024-09-20T22:15:48.086 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:47 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-20T22:15:48.087 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:47 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-20T22:15:48.636 INFO:teuthology.orchestra.run.smithi037.stdout:{"epoch":55,"num_osds":8,"num_up_osds":8,"osd_up_since":1726870547,"num_in_osds":8,"osd_in_since":1726870530,"num_remapped_pgs":1} 2024-09-20T22:15:48.637 DEBUG:teuthology.orchestra.run.smithi037:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -- ceph osd dump --format=json 2024-09-20T22:15:48.856 INFO:teuthology.orchestra.run.smithi037.stderr:Inferring config /var/lib/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/mon.smithi037/config 2024-09-20T22:15:48.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:48 smithi037 ceph-mon[23718]: purged_snaps scrub starts 2024-09-20T22:15:48.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:48 smithi037 ceph-mon[23718]: purged_snaps scrub ok 2024-09-20T22:15:48.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:48 smithi037 ceph-mon[23718]: osd.7 [v2:172.21.15.194:6824/2294923533,v1:172.21.15.194:6825/2294923533] boot 2024-09-20T22:15:48.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:48 smithi037 ceph-mon[23718]: osdmap e55: 8 total, 8 up, 8 in 2024-09-20T22:15:48.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:48 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-20T22:15:48.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:48 smithi037 ceph-mon[23718]: from='client.? 172.21.15.37:0/3421633797' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-20T22:15:48.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:48 smithi037 ceph-mon[23718]: osdmap e56: 8 total, 8 up, 8 in 2024-09-20T22:15:48.960 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:48 smithi194 ceph-mon[25957]: purged_snaps scrub starts 2024-09-20T22:15:48.960 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:48 smithi194 ceph-mon[25957]: purged_snaps scrub ok 2024-09-20T22:15:48.960 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:48 smithi194 ceph-mon[25957]: osd.7 [v2:172.21.15.194:6824/2294923533,v1:172.21.15.194:6825/2294923533] boot 2024-09-20T22:15:48.960 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:48 smithi194 ceph-mon[25957]: osdmap e55: 8 total, 8 up, 8 in 2024-09-20T22:15:48.960 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:48 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-20T22:15:48.960 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:48 smithi194 ceph-mon[25957]: from='client.? 172.21.15.37:0/3421633797' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-20T22:15:48.960 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:48 smithi194 ceph-mon[25957]: osdmap e56: 8 total, 8 up, 8 in 2024-09-20T22:15:49.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:49 smithi194 ceph-mon[25957]: pgmap v168: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-20T22:15:49.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:49 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:15:49.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:49 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-20T22:15:49.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:49 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-20T22:15:49.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:49 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-20T22:15:49.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:49 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-20T22:15:49.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:49 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:15:49.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:49 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:15:49.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:49 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:15:49.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:49 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:15:49.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:49 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:15:49.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:49 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:15:49.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:49 smithi194 ceph-mon[25957]: osdmap e57: 8 total, 8 up, 8 in 2024-09-20T22:15:49.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:49 smithi037 ceph-mon[23718]: pgmap v168: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-20T22:15:49.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:49 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:15:49.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:49 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-20T22:15:49.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:49 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-20T22:15:49.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:49 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-20T22:15:49.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:49 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-20T22:15:49.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:49 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:15:49.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:49 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:15:49.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:49 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:15:49.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:49 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:15:49.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:49 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:15:49.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:49 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:15:49.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:49 smithi037 ceph-mon[23718]: osdmap e57: 8 total, 8 up, 8 in 2024-09-20T22:15:50.704 INFO:teuthology.orchestra.run.smithi037.stdout: 2024-09-20T22:15:50.704 INFO:teuthology.orchestra.run.smithi037.stdout:{"epoch":58,"fsid":"f9a6357a-779c-11ef-baf5-efdc52797490","created":"2024-09-20T22:09:43.122584+0000","modified":"2024-09-20T22:15:50.674619+0000","last_up_change":"2024-09-20T22:15:47.685022+0000","last_in_change":"2024-09-20T22:15:30.740167+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-20T22:13:50.617543+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"03a32e66-ac08-4daf-94c6-cd32f709392f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6802","nonce":802225310},{"type":"v1","addr":"172.21.15.37:6803","nonce":802225310}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6804","nonce":802225310},{"type":"v1","addr":"172.21.15.37:6805","nonce":802225310}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6808","nonce":802225310},{"type":"v1","addr":"172.21.15.37:6809","nonce":802225310}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6806","nonce":802225310},{"type":"v1","addr":"172.21.15.37:6807","nonce":802225310}]},"public_addr":"172.21.15.37:6803/802225310","cluster_addr":"172.21.15.37:6805/802225310","heartbeat_back_addr":"172.21.15.37:6809/802225310","heartbeat_front_addr":"172.21.15.37:6807/802225310","state":["exists","up"]},{"osd":1,"uuid":"5f6c77dd-46a1-4dd2-91e0-f05faa539e5a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6810","nonce":441045715},{"type":"v1","addr":"172.21.15.37:6811","nonce":441045715}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6812","nonce":441045715},{"type":"v1","addr":"172.21.15.37:6813","nonce":441045715}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6816","nonce":441045715},{"type":"v1","addr":"172.21.15.37:6817","nonce":441045715}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6814","nonce":441045715},{"type":"v1","addr":"172.21.15.37:6815","nonce":441045715}]},"public_addr":"172.21.15.37:6811/441045715","cluster_addr":"172.21.15.37:6813/441045715","heartbeat_back_addr":"172.21.15.37:6817/441045715","heartbeat_front_addr":"172.21.15.37:6815/441045715","state":["exists","up"]},{"osd":2,"uuid":"3469715b-d6bc-4535-9a87-ea486cd8fd9a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6818","nonce":4264091112},{"type":"v1","addr":"172.21.15.37:6819","nonce":4264091112}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6820","nonce":4264091112},{"type":"v1","addr":"172.21.15.37:6821","nonce":4264091112}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6824","nonce":4264091112},{"type":"v1","addr":"172.21.15.37:6825","nonce":4264091112}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6822","nonce":4264091112},{"type":"v1","addr":"172.21.15.37:6823","nonce":4264091112}]},"public_addr":"172.21.15.37:6819/4264091112","cluster_addr":"172.21.15.37:6821/4264091112","heartbeat_back_addr":"172.21.15.37:6825/4264091112","heartbeat_front_addr":"172.21.15.37:6823/4264091112","state":["exists","up"]},{"osd":3,"uuid":"d555a027-d177-4ffa-9761-dd1a06b4c1a2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6826","nonce":4193735334},{"type":"v1","addr":"172.21.15.37:6827","nonce":4193735334}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6828","nonce":4193735334},{"type":"v1","addr":"172.21.15.37:6829","nonce":4193735334}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6832","nonce":4193735334},{"type":"v1","addr":"172.21.15.37:6833","nonce":4193735334}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6830","nonce":4193735334},{"type":"v1","addr":"172.21.15.37:6831","nonce":4193735334}]},"public_addr":"172.21.15.37:6827/4193735334","cluster_addr":"172.21.15.37:6829/4193735334","heartbeat_back_addr":"172.21.15.37:6833/4193735334","heartbeat_front_addr":"172.21.15.37:6831/4193735334","state":["exists","up"]},{"osd":4,"uuid":"268e1f18-01d9-4a1f-94b3-4cd5c6d0f843","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":4219258866},{"type":"v1","addr":"172.21.15.194:6801","nonce":4219258866}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":4219258866},{"type":"v1","addr":"172.21.15.194:6803","nonce":4219258866}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":4219258866},{"type":"v1","addr":"172.21.15.194:6807","nonce":4219258866}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":4219258866},{"type":"v1","addr":"172.21.15.194:6805","nonce":4219258866}]},"public_addr":"172.21.15.194:6801/4219258866","cluster_addr":"172.21.15.194:6803/4219258866","heartbeat_back_addr":"172.21.15.194:6807/4219258866","heartbeat_front_addr":"172.21.15.194:6805/4219258866","state":["exists","up"]},{"osd":5,"uuid":"06dd9a31-ccb8-4238-a4d8-ca6edc0ec774","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":3326488387},{"type":"v1","addr":"172.21.15.194:6809","nonce":3326488387}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":3326488387},{"type":"v1","addr":"172.21.15.194:6811","nonce":3326488387}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":3326488387},{"type":"v1","addr":"172.21.15.194:6815","nonce":3326488387}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":3326488387},{"type":"v1","addr":"172.21.15.194:6813","nonce":3326488387}]},"public_addr":"172.21.15.194:6809/3326488387","cluster_addr":"172.21.15.194:6811/3326488387","heartbeat_back_addr":"172.21.15.194:6815/3326488387","heartbeat_front_addr":"172.21.15.194:6813/3326488387","state":["exists","up"]},{"osd":6,"uuid":"2ae4c518-af42-4827-bb85-8e8a11b72ea2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1380757103},{"type":"v1","addr":"172.21.15.194:6817","nonce":1380757103}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1380757103},{"type":"v1","addr":"172.21.15.194:6819","nonce":1380757103}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1380757103},{"type":"v1","addr":"172.21.15.194:6823","nonce":1380757103}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6820","nonce":1380757103},{"type":"v1","addr":"172.21.15.194:6821","nonce":1380757103}]},"public_addr":"172.21.15.194:6817/1380757103","cluster_addr":"172.21.15.194:6819/1380757103","heartbeat_back_addr":"172.21.15.194:6823/1380757103","heartbeat_front_addr":"172.21.15.194:6821/1380757103","state":["exists","up"]},{"osd":7,"uuid":"8acd1795-badf-4632-99b1-f61a81bce2cf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6824","nonce":2294923533},{"type":"v1","addr":"172.21.15.194:6825","nonce":2294923533}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":2294923533},{"type":"v1","addr":"172.21.15.194:6827","nonce":2294923533}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":2294923533},{"type":"v1","addr":"172.21.15.194:6831","nonce":2294923533}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6828","nonce":2294923533},{"type":"v1","addr":"172.21.15.194:6829","nonce":2294923533}]},"public_addr":"172.21.15.194:6825/2294923533","cluster_addr":"172.21.15.194:6827/2294923533","heartbeat_back_addr":"172.21.15.194:6831/2294923533","heartbeat_front_addr":"172.21.15.194:6829/2294923533","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T22:12:50.105915+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T22:13:17.183241+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T22:13:47.881995+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T22:14:15.131980+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T22:14:36.011048+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T22:14:58.541927+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T22:15:22.315138+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T22:15:46.028500+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.37:6801/2084936584":"2024-09-21T22:11:38.054253+0000","172.21.15.37:6800/2084936584":"2024-09-21T22:11:38.054253+0000","172.21.15.37:0/4069020005":"2024-09-21T22:11:38.054253+0000","172.21.15.37:0/83689753":"2024-09-21T22:11:38.054253+0000","172.21.15.37:0/3350204404":"2024-09-21T22:10:46.801456+0000","172.21.15.37:6800/1118557896":"2024-09-21T22:10:46.801456+0000","172.21.15.37:0/1488839821":"2024-09-21T22:10:46.801456+0000","172.21.15.37:0/399199611":"2024-09-21T22:10:46.801456+0000","172.21.15.37:0/4061256501":"2024-09-21T22:10:16.071525+0000","172.21.15.37:0/3807137416":"2024-09-21T22:10:16.071525+0000","172.21.15.37:6800/3530173291":"2024-09-21T22:10:16.071525+0000","172.21.15.37:6801/1118557896":"2024-09-21T22:10:46.801456+0000","172.21.15.37:6801/3530173291":"2024-09-21T22:10:16.071525+0000","172.21.15.37:0/690267038":"2024-09-21T22:11:38.054253+0000","172.21.15.37:0/1383818180":"2024-09-21T22:10:16.071525+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-20T22:15:51.066 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:50 smithi037 ceph-mon[23718]: Detected new or changed devices on smithi194 2024-09-20T22:15:51.067 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:50 smithi037 ceph-mon[23718]: Adjusting osd_memory_target on smithi194 to 3984M 2024-09-20T22:15:51.067 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:50 smithi037 ceph-mon[23718]: osdmap e58: 8 total, 8 up, 8 in 2024-09-20T22:15:51.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:50 smithi194 ceph-mon[25957]: Detected new or changed devices on smithi194 2024-09-20T22:15:51.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:50 smithi194 ceph-mon[25957]: Adjusting osd_memory_target on smithi194 to 3984M 2024-09-20T22:15:51.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:50 smithi194 ceph-mon[25957]: osdmap e58: 8 total, 8 up, 8 in 2024-09-20T22:15:51.464 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-09-20T22:13:50.617543+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-09-20T22:15:51.464 DEBUG:teuthology.orchestra.run.smithi037:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -- ceph osd pool get .mgr pg_num 2024-09-20T22:15:51.688 INFO:teuthology.orchestra.run.smithi037.stderr:Inferring config /var/lib/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/mon.smithi037/config 2024-09-20T22:15:52.036 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:51 smithi037 ceph-mon[23718]: pgmap v171: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-20T22:15:52.037 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:51 smithi037 ceph-mon[23718]: from='client.? 172.21.15.37:0/1852550627' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-20T22:15:52.037 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:51 smithi037 ceph-mon[23718]: osdmap e59: 8 total, 8 up, 8 in 2024-09-20T22:15:52.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:51 smithi194 ceph-mon[25957]: pgmap v171: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-20T22:15:52.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:51 smithi194 ceph-mon[25957]: from='client.? 172.21.15.37:0/1852550627' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-20T22:15:52.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:51 smithi194 ceph-mon[25957]: osdmap e59: 8 total, 8 up, 8 in 2024-09-20T22:15:53.608 INFO:teuthology.orchestra.run.smithi037.stdout:pg_num: 1 2024-09-20T22:15:53.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:53 smithi037 ceph-mon[23718]: pgmap v174: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-20T22:15:53.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:53 smithi037 ceph-mon[23718]: from='client.? 172.21.15.37:0/3210208212' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-20T22:15:54.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:53 smithi194 ceph-mon[25957]: pgmap v174: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-20T22:15:54.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:53 smithi194 ceph-mon[25957]: from='client.? 172.21.15.37:0/3210208212' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-20T22:15:54.310 INFO:tasks.cephadm:Setting up client nodes... 2024-09-20T22:15:54.310 DEBUG:teuthology.orchestra.run.smithi037:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-09-20T22:15:56.027 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:55 smithi037 ceph-mon[23718]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 94 KiB/s, 0 objects/s recovering 2024-09-20T22:15:56.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:55 smithi194 ceph-mon[25957]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 94 KiB/s, 0 objects/s recovering 2024-09-20T22:15:56.570 INFO:teuthology.orchestra.run.smithi037.stdout:[client.0] 2024-09-20T22:15:56.570 INFO:teuthology.orchestra.run.smithi037.stdout: key = AQAc9O1mEve8IRAAaVeEj4P25sgLaYfLGvS4JQ== 2024-09-20T22:15:56.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:56 smithi037 ceph-mon[23718]: from='client.? 172.21.15.37:0/2267023888' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-20T22:15:56.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:56 smithi037 ceph-mon[23718]: from='client.? 172.21.15.37:0/2267023888' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-20T22:15:57.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:56 smithi194 ceph-mon[25957]: from='client.? 172.21.15.37:0/2267023888' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-20T22:15:57.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:56 smithi194 ceph-mon[25957]: from='client.? 172.21.15.37:0/2267023888' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-20T22:15:57.348 DEBUG:teuthology.orchestra.run.smithi037:> set -ex 2024-09-20T22:15:57.348 DEBUG:teuthology.orchestra.run.smithi037:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-09-20T22:15:57.348 DEBUG:teuthology.orchestra.run.smithi037:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-09-20T22:15:57.390 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-09-20T22:15:57.390 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-09-20T22:15:57.390 DEBUG:teuthology.orchestra.run.smithi037:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -- ceph mgr dump --format=json 2024-09-20T22:15:57.645 INFO:teuthology.orchestra.run.smithi037.stderr:Inferring config /var/lib/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/mon.smithi037/config 2024-09-20T22:15:57.989 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:57 smithi037 ceph-mon[23718]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 80 KiB/s, 0 objects/s recovering 2024-09-20T22:15:58.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:57 smithi194 ceph-mon[25957]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 80 KiB/s, 0 objects/s recovering 2024-09-20T22:15:59.440 INFO:teuthology.orchestra.run.smithi037.stdout: 2024-09-20T22:16:00.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:59 smithi194 ceph-mon[25957]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-20T22:16:00.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:15:59 smithi194 ceph-mon[25957]: from='client.? 172.21.15.37:0/2244298170' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-20T22:16:00.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:59 smithi037 ceph-mon[23718]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-20T22:16:00.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:15:59 smithi037 ceph-mon[23718]: from='client.? 172.21.15.37:0/2244298170' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-20T22:16:00.269 INFO:teuthology.orchestra.run.smithi037.stdout:{"epoch":19,"active_gid":14204,"active_name":"smithi037.lykblh","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6800","nonce":1904409710},{"type":"v1","addr":"172.21.15.37:6801","nonce":1904409710}]},"active_addr":"172.21.15.37:6801/1904409710","active_change":"2024-09-20T22:11:38.054635+0000","active_mgr_features":4540138303579357183,"available":true,"standbys":[{"gid":14222,"name":"smithi194.bkcglt","mgr_features":4540138303579357183,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.37:8443/","prometheus":"http://172.21.15.37:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"last_failure_osd_epoch":5,"active_clients":[{"addrvec":[{"type":"v2","addr":"172.21.15.37:0","nonce":4264986342}]},{"addrvec":[{"type":"v2","addr":"172.21.15.37:0","nonce":1482719532}]},{"addrvec":[{"type":"v2","addr":"172.21.15.37:0","nonce":1603681990}]},{"addrvec":[{"type":"v2","addr":"172.21.15.37:0","nonce":1008932622}]}]}} 2024-09-20T22:16:00.273 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-09-20T22:16:00.273 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-09-20T22:16:00.273 DEBUG:teuthology.orchestra.run.smithi037:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -- ceph osd dump --format=json 2024-09-20T22:16:00.493 INFO:teuthology.orchestra.run.smithi037.stderr:Inferring config /var/lib/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/mon.smithi037/config 2024-09-20T22:16:01.848 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:16:01 smithi037 ceph-mon[23718]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 54 KiB/s, 0 objects/s recovering 2024-09-20T22:16:02.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:16:01 smithi194 ceph-mon[25957]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 54 KiB/s, 0 objects/s recovering 2024-09-20T22:16:02.402 INFO:teuthology.orchestra.run.smithi037.stdout: 2024-09-20T22:16:02.402 INFO:teuthology.orchestra.run.smithi037.stdout:{"epoch":59,"fsid":"f9a6357a-779c-11ef-baf5-efdc52797490","created":"2024-09-20T22:09:43.122584+0000","modified":"2024-09-20T22:15:51.675754+0000","last_up_change":"2024-09-20T22:15:47.685022+0000","last_in_change":"2024-09-20T22:15:30.740167+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-20T22:13:50.617543+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"03a32e66-ac08-4daf-94c6-cd32f709392f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6802","nonce":802225310},{"type":"v1","addr":"172.21.15.37:6803","nonce":802225310}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6804","nonce":802225310},{"type":"v1","addr":"172.21.15.37:6805","nonce":802225310}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6808","nonce":802225310},{"type":"v1","addr":"172.21.15.37:6809","nonce":802225310}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6806","nonce":802225310},{"type":"v1","addr":"172.21.15.37:6807","nonce":802225310}]},"public_addr":"172.21.15.37:6803/802225310","cluster_addr":"172.21.15.37:6805/802225310","heartbeat_back_addr":"172.21.15.37:6809/802225310","heartbeat_front_addr":"172.21.15.37:6807/802225310","state":["exists","up"]},{"osd":1,"uuid":"5f6c77dd-46a1-4dd2-91e0-f05faa539e5a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6810","nonce":441045715},{"type":"v1","addr":"172.21.15.37:6811","nonce":441045715}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6812","nonce":441045715},{"type":"v1","addr":"172.21.15.37:6813","nonce":441045715}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6816","nonce":441045715},{"type":"v1","addr":"172.21.15.37:6817","nonce":441045715}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6814","nonce":441045715},{"type":"v1","addr":"172.21.15.37:6815","nonce":441045715}]},"public_addr":"172.21.15.37:6811/441045715","cluster_addr":"172.21.15.37:6813/441045715","heartbeat_back_addr":"172.21.15.37:6817/441045715","heartbeat_front_addr":"172.21.15.37:6815/441045715","state":["exists","up"]},{"osd":2,"uuid":"3469715b-d6bc-4535-9a87-ea486cd8fd9a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6818","nonce":4264091112},{"type":"v1","addr":"172.21.15.37:6819","nonce":4264091112}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6820","nonce":4264091112},{"type":"v1","addr":"172.21.15.37:6821","nonce":4264091112}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6824","nonce":4264091112},{"type":"v1","addr":"172.21.15.37:6825","nonce":4264091112}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6822","nonce":4264091112},{"type":"v1","addr":"172.21.15.37:6823","nonce":4264091112}]},"public_addr":"172.21.15.37:6819/4264091112","cluster_addr":"172.21.15.37:6821/4264091112","heartbeat_back_addr":"172.21.15.37:6825/4264091112","heartbeat_front_addr":"172.21.15.37:6823/4264091112","state":["exists","up"]},{"osd":3,"uuid":"d555a027-d177-4ffa-9761-dd1a06b4c1a2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6826","nonce":4193735334},{"type":"v1","addr":"172.21.15.37:6827","nonce":4193735334}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6828","nonce":4193735334},{"type":"v1","addr":"172.21.15.37:6829","nonce":4193735334}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6832","nonce":4193735334},{"type":"v1","addr":"172.21.15.37:6833","nonce":4193735334}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6830","nonce":4193735334},{"type":"v1","addr":"172.21.15.37:6831","nonce":4193735334}]},"public_addr":"172.21.15.37:6827/4193735334","cluster_addr":"172.21.15.37:6829/4193735334","heartbeat_back_addr":"172.21.15.37:6833/4193735334","heartbeat_front_addr":"172.21.15.37:6831/4193735334","state":["exists","up"]},{"osd":4,"uuid":"268e1f18-01d9-4a1f-94b3-4cd5c6d0f843","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":4219258866},{"type":"v1","addr":"172.21.15.194:6801","nonce":4219258866}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":4219258866},{"type":"v1","addr":"172.21.15.194:6803","nonce":4219258866}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":4219258866},{"type":"v1","addr":"172.21.15.194:6807","nonce":4219258866}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":4219258866},{"type":"v1","addr":"172.21.15.194:6805","nonce":4219258866}]},"public_addr":"172.21.15.194:6801/4219258866","cluster_addr":"172.21.15.194:6803/4219258866","heartbeat_back_addr":"172.21.15.194:6807/4219258866","heartbeat_front_addr":"172.21.15.194:6805/4219258866","state":["exists","up"]},{"osd":5,"uuid":"06dd9a31-ccb8-4238-a4d8-ca6edc0ec774","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":3326488387},{"type":"v1","addr":"172.21.15.194:6809","nonce":3326488387}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":3326488387},{"type":"v1","addr":"172.21.15.194:6811","nonce":3326488387}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":3326488387},{"type":"v1","addr":"172.21.15.194:6815","nonce":3326488387}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":3326488387},{"type":"v1","addr":"172.21.15.194:6813","nonce":3326488387}]},"public_addr":"172.21.15.194:6809/3326488387","cluster_addr":"172.21.15.194:6811/3326488387","heartbeat_back_addr":"172.21.15.194:6815/3326488387","heartbeat_front_addr":"172.21.15.194:6813/3326488387","state":["exists","up"]},{"osd":6,"uuid":"2ae4c518-af42-4827-bb85-8e8a11b72ea2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1380757103},{"type":"v1","addr":"172.21.15.194:6817","nonce":1380757103}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1380757103},{"type":"v1","addr":"172.21.15.194:6819","nonce":1380757103}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1380757103},{"type":"v1","addr":"172.21.15.194:6823","nonce":1380757103}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6820","nonce":1380757103},{"type":"v1","addr":"172.21.15.194:6821","nonce":1380757103}]},"public_addr":"172.21.15.194:6817/1380757103","cluster_addr":"172.21.15.194:6819/1380757103","heartbeat_back_addr":"172.21.15.194:6823/1380757103","heartbeat_front_addr":"172.21.15.194:6821/1380757103","state":["exists","up"]},{"osd":7,"uuid":"8acd1795-badf-4632-99b1-f61a81bce2cf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6824","nonce":2294923533},{"type":"v1","addr":"172.21.15.194:6825","nonce":2294923533}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":2294923533},{"type":"v1","addr":"172.21.15.194:6827","nonce":2294923533}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":2294923533},{"type":"v1","addr":"172.21.15.194:6831","nonce":2294923533}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6828","nonce":2294923533},{"type":"v1","addr":"172.21.15.194:6829","nonce":2294923533}]},"public_addr":"172.21.15.194:6825/2294923533","cluster_addr":"172.21.15.194:6827/2294923533","heartbeat_back_addr":"172.21.15.194:6831/2294923533","heartbeat_front_addr":"172.21.15.194:6829/2294923533","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T22:12:50.105915+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T22:13:17.183241+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T22:13:47.881995+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T22:14:15.131980+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T22:14:36.011048+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T22:14:58.541927+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T22:15:22.315138+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T22:15:46.028500+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.37:6801/2084936584":"2024-09-21T22:11:38.054253+0000","172.21.15.37:6800/2084936584":"2024-09-21T22:11:38.054253+0000","172.21.15.37:0/4069020005":"2024-09-21T22:11:38.054253+0000","172.21.15.37:0/83689753":"2024-09-21T22:11:38.054253+0000","172.21.15.37:0/3350204404":"2024-09-21T22:10:46.801456+0000","172.21.15.37:6800/1118557896":"2024-09-21T22:10:46.801456+0000","172.21.15.37:0/1488839821":"2024-09-21T22:10:46.801456+0000","172.21.15.37:0/399199611":"2024-09-21T22:10:46.801456+0000","172.21.15.37:0/4061256501":"2024-09-21T22:10:16.071525+0000","172.21.15.37:0/3807137416":"2024-09-21T22:10:16.071525+0000","172.21.15.37:6800/3530173291":"2024-09-21T22:10:16.071525+0000","172.21.15.37:6801/1118557896":"2024-09-21T22:10:46.801456+0000","172.21.15.37:6801/3530173291":"2024-09-21T22:10:16.071525+0000","172.21.15.37:0/690267038":"2024-09-21T22:11:38.054253+0000","172.21.15.37:0/1383818180":"2024-09-21T22:10:16.071525+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-20T22:16:03.029 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:16:02 smithi037 ceph-mon[23718]: from='client.? 172.21.15.37:0/2569312781' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-20T22:16:03.059 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-09-20T22:16:03.060 DEBUG:teuthology.orchestra.run.smithi037:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -- ceph osd dump --format=json 2024-09-20T22:16:03.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:16:02 smithi194 ceph-mon[25957]: from='client.? 172.21.15.37:0/2569312781' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-20T22:16:03.281 INFO:teuthology.orchestra.run.smithi037.stderr:Inferring config /var/lib/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/mon.smithi037/config 2024-09-20T22:16:04.044 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:16:03 smithi037 ceph-mon[23718]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 49 KiB/s, 0 objects/s recovering 2024-09-20T22:16:04.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:16:03 smithi194 ceph-mon[25957]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 49 KiB/s, 0 objects/s recovering 2024-09-20T22:16:05.143 INFO:teuthology.orchestra.run.smithi037.stdout: 2024-09-20T22:16:05.143 INFO:teuthology.orchestra.run.smithi037.stdout:{"epoch":59,"fsid":"f9a6357a-779c-11ef-baf5-efdc52797490","created":"2024-09-20T22:09:43.122584+0000","modified":"2024-09-20T22:15:51.675754+0000","last_up_change":"2024-09-20T22:15:47.685022+0000","last_in_change":"2024-09-20T22:15:30.740167+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-20T22:13:50.617543+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"03a32e66-ac08-4daf-94c6-cd32f709392f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6802","nonce":802225310},{"type":"v1","addr":"172.21.15.37:6803","nonce":802225310}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6804","nonce":802225310},{"type":"v1","addr":"172.21.15.37:6805","nonce":802225310}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6808","nonce":802225310},{"type":"v1","addr":"172.21.15.37:6809","nonce":802225310}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6806","nonce":802225310},{"type":"v1","addr":"172.21.15.37:6807","nonce":802225310}]},"public_addr":"172.21.15.37:6803/802225310","cluster_addr":"172.21.15.37:6805/802225310","heartbeat_back_addr":"172.21.15.37:6809/802225310","heartbeat_front_addr":"172.21.15.37:6807/802225310","state":["exists","up"]},{"osd":1,"uuid":"5f6c77dd-46a1-4dd2-91e0-f05faa539e5a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6810","nonce":441045715},{"type":"v1","addr":"172.21.15.37:6811","nonce":441045715}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6812","nonce":441045715},{"type":"v1","addr":"172.21.15.37:6813","nonce":441045715}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6816","nonce":441045715},{"type":"v1","addr":"172.21.15.37:6817","nonce":441045715}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6814","nonce":441045715},{"type":"v1","addr":"172.21.15.37:6815","nonce":441045715}]},"public_addr":"172.21.15.37:6811/441045715","cluster_addr":"172.21.15.37:6813/441045715","heartbeat_back_addr":"172.21.15.37:6817/441045715","heartbeat_front_addr":"172.21.15.37:6815/441045715","state":["exists","up"]},{"osd":2,"uuid":"3469715b-d6bc-4535-9a87-ea486cd8fd9a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6818","nonce":4264091112},{"type":"v1","addr":"172.21.15.37:6819","nonce":4264091112}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6820","nonce":4264091112},{"type":"v1","addr":"172.21.15.37:6821","nonce":4264091112}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6824","nonce":4264091112},{"type":"v1","addr":"172.21.15.37:6825","nonce":4264091112}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6822","nonce":4264091112},{"type":"v1","addr":"172.21.15.37:6823","nonce":4264091112}]},"public_addr":"172.21.15.37:6819/4264091112","cluster_addr":"172.21.15.37:6821/4264091112","heartbeat_back_addr":"172.21.15.37:6825/4264091112","heartbeat_front_addr":"172.21.15.37:6823/4264091112","state":["exists","up"]},{"osd":3,"uuid":"d555a027-d177-4ffa-9761-dd1a06b4c1a2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6826","nonce":4193735334},{"type":"v1","addr":"172.21.15.37:6827","nonce":4193735334}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6828","nonce":4193735334},{"type":"v1","addr":"172.21.15.37:6829","nonce":4193735334}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6832","nonce":4193735334},{"type":"v1","addr":"172.21.15.37:6833","nonce":4193735334}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6830","nonce":4193735334},{"type":"v1","addr":"172.21.15.37:6831","nonce":4193735334}]},"public_addr":"172.21.15.37:6827/4193735334","cluster_addr":"172.21.15.37:6829/4193735334","heartbeat_back_addr":"172.21.15.37:6833/4193735334","heartbeat_front_addr":"172.21.15.37:6831/4193735334","state":["exists","up"]},{"osd":4,"uuid":"268e1f18-01d9-4a1f-94b3-4cd5c6d0f843","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":4219258866},{"type":"v1","addr":"172.21.15.194:6801","nonce":4219258866}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":4219258866},{"type":"v1","addr":"172.21.15.194:6803","nonce":4219258866}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":4219258866},{"type":"v1","addr":"172.21.15.194:6807","nonce":4219258866}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":4219258866},{"type":"v1","addr":"172.21.15.194:6805","nonce":4219258866}]},"public_addr":"172.21.15.194:6801/4219258866","cluster_addr":"172.21.15.194:6803/4219258866","heartbeat_back_addr":"172.21.15.194:6807/4219258866","heartbeat_front_addr":"172.21.15.194:6805/4219258866","state":["exists","up"]},{"osd":5,"uuid":"06dd9a31-ccb8-4238-a4d8-ca6edc0ec774","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":3326488387},{"type":"v1","addr":"172.21.15.194:6809","nonce":3326488387}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":3326488387},{"type":"v1","addr":"172.21.15.194:6811","nonce":3326488387}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":3326488387},{"type":"v1","addr":"172.21.15.194:6815","nonce":3326488387}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":3326488387},{"type":"v1","addr":"172.21.15.194:6813","nonce":3326488387}]},"public_addr":"172.21.15.194:6809/3326488387","cluster_addr":"172.21.15.194:6811/3326488387","heartbeat_back_addr":"172.21.15.194:6815/3326488387","heartbeat_front_addr":"172.21.15.194:6813/3326488387","state":["exists","up"]},{"osd":6,"uuid":"2ae4c518-af42-4827-bb85-8e8a11b72ea2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1380757103},{"type":"v1","addr":"172.21.15.194:6817","nonce":1380757103}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1380757103},{"type":"v1","addr":"172.21.15.194:6819","nonce":1380757103}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1380757103},{"type":"v1","addr":"172.21.15.194:6823","nonce":1380757103}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6820","nonce":1380757103},{"type":"v1","addr":"172.21.15.194:6821","nonce":1380757103}]},"public_addr":"172.21.15.194:6817/1380757103","cluster_addr":"172.21.15.194:6819/1380757103","heartbeat_back_addr":"172.21.15.194:6823/1380757103","heartbeat_front_addr":"172.21.15.194:6821/1380757103","state":["exists","up"]},{"osd":7,"uuid":"8acd1795-badf-4632-99b1-f61a81bce2cf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6824","nonce":2294923533},{"type":"v1","addr":"172.21.15.194:6825","nonce":2294923533}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":2294923533},{"type":"v1","addr":"172.21.15.194:6827","nonce":2294923533}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":2294923533},{"type":"v1","addr":"172.21.15.194:6831","nonce":2294923533}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6828","nonce":2294923533},{"type":"v1","addr":"172.21.15.194:6829","nonce":2294923533}]},"public_addr":"172.21.15.194:6825/2294923533","cluster_addr":"172.21.15.194:6827/2294923533","heartbeat_back_addr":"172.21.15.194:6831/2294923533","heartbeat_front_addr":"172.21.15.194:6829/2294923533","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T22:12:50.105915+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T22:13:17.183241+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T22:13:47.881995+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T22:14:15.131980+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T22:14:36.011048+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T22:14:58.541927+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T22:15:22.315138+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T22:15:46.028500+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.37:6801/2084936584":"2024-09-21T22:11:38.054253+0000","172.21.15.37:6800/2084936584":"2024-09-21T22:11:38.054253+0000","172.21.15.37:0/4069020005":"2024-09-21T22:11:38.054253+0000","172.21.15.37:0/83689753":"2024-09-21T22:11:38.054253+0000","172.21.15.37:0/3350204404":"2024-09-21T22:10:46.801456+0000","172.21.15.37:6800/1118557896":"2024-09-21T22:10:46.801456+0000","172.21.15.37:0/1488839821":"2024-09-21T22:10:46.801456+0000","172.21.15.37:0/399199611":"2024-09-21T22:10:46.801456+0000","172.21.15.37:0/4061256501":"2024-09-21T22:10:16.071525+0000","172.21.15.37:0/3807137416":"2024-09-21T22:10:16.071525+0000","172.21.15.37:6800/3530173291":"2024-09-21T22:10:16.071525+0000","172.21.15.37:6801/1118557896":"2024-09-21T22:10:46.801456+0000","172.21.15.37:6801/3530173291":"2024-09-21T22:10:16.071525+0000","172.21.15.37:0/690267038":"2024-09-21T22:11:38.054253+0000","172.21.15.37:0/1383818180":"2024-09-21T22:10:16.071525+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-20T22:16:05.777 DEBUG:teuthology.orchestra.run.smithi037:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -- ceph tell osd.0 flush_pg_stats 2024-09-20T22:16:05.777 DEBUG:teuthology.orchestra.run.smithi037:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -- ceph tell osd.1 flush_pg_stats 2024-09-20T22:16:05.778 DEBUG:teuthology.orchestra.run.smithi037:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -- ceph tell osd.2 flush_pg_stats 2024-09-20T22:16:05.778 DEBUG:teuthology.orchestra.run.smithi037:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -- ceph tell osd.3 flush_pg_stats 2024-09-20T22:16:05.779 DEBUG:teuthology.orchestra.run.smithi037:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -- ceph tell osd.4 flush_pg_stats 2024-09-20T22:16:05.779 DEBUG:teuthology.orchestra.run.smithi037:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -- ceph tell osd.5 flush_pg_stats 2024-09-20T22:16:05.779 DEBUG:teuthology.orchestra.run.smithi037:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -- ceph tell osd.6 flush_pg_stats 2024-09-20T22:16:05.780 DEBUG:teuthology.orchestra.run.smithi037:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -- ceph tell osd.7 flush_pg_stats 2024-09-20T22:16:06.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:16:05 smithi194 ceph-mon[25957]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s, 0 objects/s recovering 2024-09-20T22:16:06.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:16:05 smithi194 ceph-mon[25957]: from='client.? 172.21.15.37:0/2224424446' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-20T22:16:06.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:16:05 smithi037 ceph-mon[23718]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s, 0 objects/s recovering 2024-09-20T22:16:06.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:16:05 smithi037 ceph-mon[23718]: from='client.? 172.21.15.37:0/2224424446' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-20T22:16:06.264 INFO:teuthology.orchestra.run.smithi037.stderr:Inferring config /var/lib/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/mon.smithi037/config 2024-09-20T22:16:06.265 INFO:teuthology.orchestra.run.smithi037.stderr:Inferring config /var/lib/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/mon.smithi037/config 2024-09-20T22:16:06.265 INFO:teuthology.orchestra.run.smithi037.stderr:Inferring config /var/lib/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/mon.smithi037/config 2024-09-20T22:16:06.268 INFO:teuthology.orchestra.run.smithi037.stderr:Inferring config /var/lib/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/mon.smithi037/config 2024-09-20T22:16:06.270 INFO:teuthology.orchestra.run.smithi037.stderr:Inferring config /var/lib/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/mon.smithi037/config 2024-09-20T22:16:06.275 INFO:teuthology.orchestra.run.smithi037.stderr:Inferring config /var/lib/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/mon.smithi037/config 2024-09-20T22:16:06.275 INFO:teuthology.orchestra.run.smithi037.stderr:Inferring config /var/lib/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/mon.smithi037/config 2024-09-20T22:16:06.307 INFO:teuthology.orchestra.run.smithi037.stderr:Inferring config /var/lib/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/mon.smithi037/config 2024-09-20T22:16:08.064 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:16:07 smithi037 ceph-mon[23718]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T22:16:08.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:16:07 smithi194 ceph-mon[25957]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T22:16:09.808 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:16:09 smithi037 ceph-mon[23718]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T22:16:10.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:16:09 smithi194 ceph-mon[25957]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T22:16:11.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:16:11 smithi037 ceph-mon[23718]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T22:16:12.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:16:11 smithi194 ceph-mon[25957]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T22:16:13.792 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:16:13 smithi037 ceph-mon[23718]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T22:16:14.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:16:13 smithi194 ceph-mon[25957]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T22:16:15.892 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:16:15 smithi037 ceph-mon[23718]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T22:16:16.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:16:15 smithi194 ceph-mon[25957]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T22:16:16.198 INFO:teuthology.orchestra.run.smithi037.stdout:167503724562 2024-09-20T22:16:16.198 DEBUG:teuthology.orchestra.run.smithi037:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -- ceph osd last-stat-seq osd.5 2024-09-20T22:16:17.205 INFO:teuthology.orchestra.run.smithi037.stdout:60129542182 2024-09-20T22:16:17.205 DEBUG:teuthology.orchestra.run.smithi037:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -- ceph osd last-stat-seq osd.1 2024-09-20T22:16:18.052 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:16:17 smithi037 ceph-mon[23718]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T22:16:18.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:16:17 smithi194 ceph-mon[25957]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T22:16:19.833 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:16:19 smithi037 ceph-mon[23718]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T22:16:20.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:16:19 smithi194 ceph-mon[25957]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T22:16:21.312 INFO:teuthology.orchestra.run.smithi037.stdout:133143986198 2024-09-20T22:16:21.312 DEBUG:teuthology.orchestra.run.smithi037:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -- ceph osd last-stat-seq osd.4 2024-09-20T22:16:21.632 INFO:teuthology.orchestra.run.smithi037.stderr:Inferring config /var/lib/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/mon.smithi037/config 2024-09-20T22:16:21.660 INFO:teuthology.orchestra.run.smithi037.stdout:201863462926 2024-09-20T22:16:21.660 DEBUG:teuthology.orchestra.run.smithi037:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -- ceph osd last-stat-seq osd.6 2024-09-20T22:16:21.878 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:16:21 smithi037 ceph-mon[23718]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T22:16:22.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:16:21 smithi194 ceph-mon[25957]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T22:16:22.646 INFO:teuthology.orchestra.run.smithi037.stdout:111669149724 2024-09-20T22:16:22.647 DEBUG:teuthology.orchestra.run.smithi037:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -- ceph osd last-stat-seq osd.3 2024-09-20T22:16:22.997 INFO:teuthology.orchestra.run.smithi037.stdout:38654705709 2024-09-20T22:16:22.998 DEBUG:teuthology.orchestra.run.smithi037:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -- ceph osd last-stat-seq osd.0 2024-09-20T22:16:23.594 INFO:teuthology.orchestra.run.smithi037.stdout:236223201289 2024-09-20T22:16:23.595 DEBUG:teuthology.orchestra.run.smithi037:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -- ceph osd last-stat-seq osd.7 2024-09-20T22:16:23.617 INFO:teuthology.orchestra.run.smithi037.stderr:Inferring config /var/lib/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/mon.smithi037/config 2024-09-20T22:16:23.618 INFO:teuthology.orchestra.run.smithi037.stderr:Inferring config /var/lib/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/mon.smithi037/config 2024-09-20T22:16:23.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:16:23 smithi037 ceph-mon[23718]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T22:16:24.062 INFO:teuthology.orchestra.run.smithi037.stdout:81604378657 2024-09-20T22:16:24.062 DEBUG:teuthology.orchestra.run.smithi037:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -- ceph osd last-stat-seq osd.2 2024-09-20T22:16:24.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:16:23 smithi194 ceph-mon[25957]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T22:16:26.023 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:16:25 smithi037 ceph-mon[23718]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T22:16:26.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:16:25 smithi194 ceph-mon[25957]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T22:16:26.412 INFO:teuthology.orchestra.run.smithi037.stdout:133143986200 2024-09-20T22:16:26.947 INFO:teuthology.orchestra.run.smithi037.stderr:Inferring config /var/lib/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/mon.smithi037/config 2024-09-20T22:16:26.947 INFO:teuthology.orchestra.run.smithi037.stderr:Inferring config /var/lib/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/mon.smithi037/config 2024-09-20T22:16:27.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:16:26 smithi194 ceph-mon[25957]: from='client.? 172.21.15.37:0/2977588971' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-20T22:16:27.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:16:26 smithi037 ceph-mon[23718]: from='client.? 172.21.15.37:0/2977588971' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-20T22:16:27.797 INFO:teuthology.orchestra.run.smithi037.stdout:60129542184 2024-09-20T22:16:28.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:16:27 smithi194 ceph-mon[25957]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T22:16:28.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:16:27 smithi037 ceph-mon[23718]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T22:16:28.423 INFO:tasks.cephadm.ceph_manager.ceph:need seq 60129542182 got 60129542184 for osd.1 2024-09-20T22:16:28.423 DEBUG:teuthology.parallel:result is None 2024-09-20T22:16:28.609 INFO:teuthology.orchestra.run.smithi037.stderr:Inferring config /var/lib/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/mon.smithi037/config 2024-09-20T22:16:28.610 INFO:teuthology.orchestra.run.smithi037.stderr:Inferring config /var/lib/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/mon.smithi037/config 2024-09-20T22:16:28.610 INFO:teuthology.orchestra.run.smithi037.stderr:Inferring config /var/lib/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/mon.smithi037/config 2024-09-20T22:16:28.963 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:16:28 smithi037 ceph-mon[23718]: from='client.? 172.21.15.37:0/157046210' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-20T22:16:29.018 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986198 got 133143986200 for osd.4 2024-09-20T22:16:29.019 DEBUG:teuthology.parallel:result is None 2024-09-20T22:16:29.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:16:28 smithi194 ceph-mon[25957]: from='client.? 172.21.15.37:0/157046210' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-20T22:16:29.911 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:16:29 smithi037 ceph-mon[23718]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T22:16:30.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:16:29 smithi194 ceph-mon[25957]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T22:16:30.970 INFO:teuthology.orchestra.run.smithi037.stdout:167503724565 2024-09-20T22:16:32.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:16:31 smithi194 ceph-mon[25957]: pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T22:16:32.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:16:31 smithi194 ceph-mon[25957]: from='client.? 172.21.15.37:0/231659762' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-20T22:16:32.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:16:31 smithi037 ceph-mon[23718]: pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T22:16:32.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:16:31 smithi037 ceph-mon[23718]: from='client.? 172.21.15.37:0/231659762' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-20T22:16:34.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:16:33 smithi194 ceph-mon[25957]: pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T22:16:34.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:16:33 smithi037 ceph-mon[23718]: pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T22:16:34.332 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724562 got 167503724565 for osd.5 2024-09-20T22:16:34.332 DEBUG:teuthology.parallel:result is None 2024-09-20T22:16:35.711 INFO:teuthology.orchestra.run.smithi037.stdout:236223201292 2024-09-20T22:16:35.956 INFO:teuthology.orchestra.run.smithi037.stdout:111669149727 2024-09-20T22:16:36.058 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:16:35 smithi037 ceph-mon[23718]: pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T22:16:36.058 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:16:35 smithi037 ceph-mon[23718]: from='client.? 172.21.15.37:0/75130576' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-20T22:16:36.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:16:35 smithi194 ceph-mon[25957]: pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T22:16:36.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:16:35 smithi194 ceph-mon[25957]: from='client.? 172.21.15.37:0/75130576' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-20T22:16:36.390 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201289 got 236223201292 for osd.7 2024-09-20T22:16:36.390 DEBUG:teuthology.parallel:result is None 2024-09-20T22:16:36.994 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:16:36 smithi037 ceph-mon[23718]: from='client.? 172.21.15.37:0/3027979930' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-20T22:16:37.020 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149724 got 111669149727 for osd.3 2024-09-20T22:16:37.020 DEBUG:teuthology.parallel:result is None 2024-09-20T22:16:37.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:16:36 smithi194 ceph-mon[25957]: from='client.? 172.21.15.37:0/3027979930' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-20T22:16:37.686 INFO:teuthology.orchestra.run.smithi037.stdout:38654705712 2024-09-20T22:16:37.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:16:37 smithi037 ceph-mon[23718]: pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T22:16:37.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:16:37 smithi037 ceph-mon[23718]: from='client.? 172.21.15.37:0/2449342929' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-20T22:16:38.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:16:37 smithi194 ceph-mon[25957]: pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T22:16:38.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:16:37 smithi194 ceph-mon[25957]: from='client.? 172.21.15.37:0/2449342929' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-20T22:16:38.697 INFO:teuthology.orchestra.run.smithi037.stdout:201863462929 2024-09-20T22:16:38.891 INFO:teuthology.orchestra.run.smithi037.stdout:81604378661 2024-09-20T22:16:38.891 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:16:38 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi037.lykblh/mirror_snapshot_schedule"}]: dispatch 2024-09-20T22:16:38.891 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:16:38 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi037.lykblh/trash_purge_schedule"}]: dispatch 2024-09-20T22:16:38.891 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:16:38 smithi037 ceph-mon[23718]: from='client.? 172.21.15.37:0/1845800828' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-20T22:16:39.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:16:38 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi037.lykblh/mirror_snapshot_schedule"}]: dispatch 2024-09-20T22:16:39.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:16:38 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi037.lykblh/trash_purge_schedule"}]: dispatch 2024-09-20T22:16:39.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:16:38 smithi194 ceph-mon[25957]: from='client.? 172.21.15.37:0/1845800828' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-20T22:16:39.522 INFO:tasks.cephadm.ceph_manager.ceph:need seq 81604378657 got 81604378661 for osd.2 2024-09-20T22:16:39.522 DEBUG:teuthology.parallel:result is None 2024-09-20T22:16:40.091 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:16:39 smithi037 ceph-mon[23718]: pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T22:16:40.091 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:16:39 smithi037 ceph-mon[23718]: from='client.? 172.21.15.37:0/1361008306' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-20T22:16:40.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:16:39 smithi194 ceph-mon[25957]: pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T22:16:40.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:16:39 smithi194 ceph-mon[25957]: from='client.? 172.21.15.37:0/1361008306' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-20T22:16:40.121 INFO:tasks.cephadm.ceph_manager.ceph:need seq 38654705709 got 38654705712 for osd.0 2024-09-20T22:16:40.121 DEBUG:teuthology.parallel:result is None 2024-09-20T22:16:40.556 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863462926 got 201863462929 for osd.6 2024-09-20T22:16:40.556 DEBUG:teuthology.parallel:result is None 2024-09-20T22:16:40.556 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-09-20T22:16:40.556 DEBUG:teuthology.orchestra.run.smithi037:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -- ceph pg dump --format=json 2024-09-20T22:16:40.782 INFO:teuthology.orchestra.run.smithi037.stderr:Inferring config /var/lib/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/mon.smithi037/config 2024-09-20T22:16:42.059 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:16:41 smithi037 ceph-mon[23718]: pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T22:16:42.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:16:41 smithi194 ceph-mon[25957]: pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T22:16:42.567 INFO:teuthology.orchestra.run.smithi037.stdout: 2024-09-20T22:16:42.568 INFO:teuthology.orchestra.run.smithi037.stderr:dumped all 2024-09-20T22:16:43.193 INFO:teuthology.orchestra.run.smithi037.stdout:{"pg_ready":true,"pg_map":{"version":199,"stamp":"2024-09-20T22:16:42.161895+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48928,"kb_used_data":6304,"kb_used_omap":0,"kb_used_meta":42560,"kb_avail":749912288,"statfs":{"total":767960285184,"available":767910182912,"internally_reserved":0,"allocated":6455296,"data_stored":3546884,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43581440},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002161"},"pg_stats":[{"pgid":"1.0","version":"22'117","reported_seq":226,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-20T22:15:51.682016+0000","last_change":"2024-09-20T22:15:51.682016+0000","last_active":"2024-09-20T22:15:51.682016+0000","last_peered":"2024-09-20T22:15:51.682016+0000","last_clean":"2024-09-20T22:15:51.682016+0000","last_became_active":"2024-09-20T22:15:51.681567+0000","last_became_peered":"2024-09-20T22:15:51.681567+0000","last_unstale":"2024-09-20T22:15:51.682016+0000","last_undegraded":"2024-09-20T22:15:51.682016+0000","last_fullsized":"2024-09-20T22:15:51.682016+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-20T22:13:50.793780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-20T22:13:50.793780+0000","last_clean_scrub_stamp":"2024-09-20T22:13:50.793780+0000","objects_scrubbed":0,"log_size":117,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-21T23:07:56.203597+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3727360,"data_stored":3702496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201293,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6048,"kb_used_data":1112,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739104,"statfs":{"total":95995035648,"available":95988842496,"internally_reserved":0,"allocated":1138688,"data_stored":773838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.63300000000000001}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.55900000000000005}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.68000000000000005}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57899999999999996}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54700000000000004}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.443}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.61599999999999999}]}]},{"osd":6,"up_from":47,"seq":201863462930,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6168,"kb_used_data":1104,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93738984,"statfs":{"total":95995035648,"available":95988719616,"internally_reserved":0,"allocated":1130496,"data_stored":773308,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 20 22:16:24 2024","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.505,"15min":0.505},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.38},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.60199999999999998}]},{"osd":1,"last update":"Fri Sep 20 22:16:24 2024","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.47199999999999998},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.45300000000000001}]},{"osd":2,"last update":"Fri Sep 20 22:16:24 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.86399999999999999,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.98499999999999999,"5min":0.98499999999999999,"15min":0.98499999999999999},"last":0.52200000000000002}]},{"osd":3,"last update":"Fri Sep 20 22:16:24 2024","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.495},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.57999999999999996}]},{"osd":4,"last update":"Fri Sep 20 22:16:24 2024","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"last":0.42399999999999999},{"interface":"front","average":{"1min":0.48999999999999999,"5min":0.48999999999999999,"15min":0.48999999999999999},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"last":0.53400000000000003}]},{"osd":5,"last update":"Fri Sep 20 22:16:24 2024","interfaces":[{"interface":"back","average":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"last":0.622},{"interface":"front","average":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.58799999999999997}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56000000000000005}]}]},{"osd":1,"up_from":14,"seq":60129542187,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6364,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738788,"statfs":{"total":95995035648,"available":95988518912,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 20 22:16:24 2024","interfaces":[{"interface":"back","average":{"1min":0.36899999999999999,"5min":0.34000000000000002,"15min":0.33000000000000002},"min":{"1min":0.25900000000000001,"5min":0.222,"15min":0.222},"max":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.40000000000000002,"5min":0.35899999999999999,"15min":0.34100000000000003},"min":{"1min":0.28100000000000003,"5min":0.221,"15min":0.221},"max":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"last":0.34300000000000003}]},{"osd":2,"last update":"Fri Sep 20 22:15:55 2024","interfaces":[{"interface":"back","average":{"1min":0.39700000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"min":{"1min":0.28699999999999998,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.65100000000000002,"5min":0.746,"15min":0.746},"last":0.35999999999999999},{"interface":"front","average":{"1min":0.44600000000000001,"5min":0.41599999999999998,"15min":0.41099999999999998},"min":{"1min":0.34999999999999998,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":0.40500000000000003}]},{"osd":3,"last update":"Fri Sep 20 22:16:24 2024","interfaces":[{"interface":"back","average":{"1min":0.46700000000000003,"5min":0.47199999999999998,"15min":0.47299999999999998},"min":{"1min":0.33600000000000002,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.501,"5min":0.495,"15min":0.49399999999999999},"min":{"1min":0.33700000000000002,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.53100000000000003}]},{"osd":4,"last update":"Fri Sep 20 22:15:39 2024","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":0.90200000000000002,"5min":0.90200000000000002,"15min":0.90200000000000002},"last":0.52000000000000002}]},{"osd":5,"last update":"Fri Sep 20 22:16:01 2024","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.78100000000000003,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.59199999999999997},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.46899999999999997}]},{"osd":6,"last update":"Fri Sep 20 22:16:24 2024","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":0.624,"15min":0.624},"min":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"max":{"1min":0.84499999999999997,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.501},{"interface":"front","average":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"min":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"max":{"1min":0.84799999999999998,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.61099999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56799999999999995}]}]},{"osd":0,"up_from":9,"seq":38654705713,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6884,"kb_used_data":1116,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738268,"statfs":{"total":95995035648,"available":95987986432,"internally_reserved":0,"allocated":1142784,"data_stored":774161,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Sep 20 22:16:28 2024","interfaces":[{"interface":"back","average":{"1min":0.40200000000000002,"5min":0.34799999999999998,"15min":0.32000000000000001},"min":{"1min":0.248,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.58199999999999996,"5min":0.627,"15min":0.627},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.40400000000000003,"5min":0.34599999999999997,"15min":0.317},"min":{"1min":0.26500000000000001,"5min":0.224,"15min":0.224},"max":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"last":0.38}]},{"osd":2,"last update":"Fri Sep 20 22:15:54 2024","interfaces":[{"interface":"back","average":{"1min":0.45900000000000002,"5min":0.38600000000000001,"15min":0.374},"min":{"1min":0.29899999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.45000000000000001,"5min":0.38200000000000001,"15min":0.37},"min":{"1min":0.33200000000000002,"5min":0.251,"15min":0.251},"max":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.436}]},{"osd":3,"last update":"Fri Sep 20 22:16:24 2024","interfaces":[{"interface":"back","average":{"1min":0.435,"5min":0.46200000000000002,"15min":0.46600000000000003},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.57399999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.46899999999999997,"5min":0.49299999999999999,"15min":0.497},"min":{"1min":0.35399999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.46500000000000002}]},{"osd":4,"last update":"Fri Sep 20 22:15:38 2024","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":0.62,"15min":0.62},"min":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"max":{"1min":0.99399999999999999,"5min":0.99399999999999999,"15min":0.99399999999999999},"last":0.56299999999999994},{"interface":"front","average":{"1min":0.622,"5min":0.622,"15min":0.622},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.98299999999999998,"5min":0.98299999999999998,"15min":0.98299999999999998},"last":0.42499999999999999}]},{"osd":5,"last update":"Fri Sep 20 22:16:00 2024","interfaces":[{"interface":"back","average":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"min":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"max":{"1min":0.90500000000000003,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.65100000000000002},{"interface":"front","average":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"min":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":0.91000000000000003,"5min":0.91000000000000003,"15min":0.91000000000000003},"last":0.50800000000000001}]},{"osd":6,"last update":"Fri Sep 20 22:16:24 2024","interfaces":[{"interface":"back","average":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.49399999999999999},{"interface":"front","average":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"min":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.53800000000000003}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.68200000000000005}]}]},{"osd":2,"up_from":19,"seq":81604378662,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6236,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5632,"kb_avail":93738916,"statfs":{"total":95995035648,"available":95988649984,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5767168},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 20 22:15:52 2024","interfaces":[{"interface":"back","average":{"1min":0.45700000000000002,"5min":0.39000000000000001,"15min":0.379},"min":{"1min":0.29699999999999999,"5min":0.216,"15min":0.216},"max":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.23300000000000001},{"interface":"front","average":{"1min":0.46200000000000002,"5min":0.41499999999999998,"15min":0.40699999999999997},"min":{"1min":0.27700000000000002,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.32900000000000001}]},{"osd":1,"last update":"Fri Sep 20 22:15:52 2024","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":0.38600000000000001,"15min":0.36799999999999999},"min":{"1min":0.26400000000000001,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.27000000000000002},{"interface":"front","average":{"1min":0.48899999999999999,"5min":0.41199999999999998,"15min":0.39900000000000002},"min":{"1min":0.28399999999999997,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.39800000000000002}]},{"osd":3,"last update":"Fri Sep 20 22:16:20 2024","interfaces":[{"interface":"back","average":{"1min":0.46899999999999997,"5min":0.48399999999999999,"15min":0.48599999999999999},"min":{"1min":0.317,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.35699999999999998},{"interface":"front","average":{"1min":0.42999999999999999,"5min":0.46500000000000002,"15min":0.47099999999999997},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"last":0.29299999999999998}]},{"osd":4,"last update":"Fri Sep 20 22:15:40 2024","interfaces":[{"interface":"back","average":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"min":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":1.121,"5min":1.121,"15min":1.121},"last":0.61799999999999999},{"interface":"front","average":{"1min":0.69399999999999995,"5min":0.69399999999999995,"15min":0.69399999999999995},"min":{"1min":0.47099999999999997,"5min":0.47099999999999997,"15min":0.47099999999999997},"max":{"1min":1.1399999999999999,"5min":1.1399999999999999,"15min":1.1399999999999999},"last":0.64800000000000002}]},{"osd":5,"last update":"Fri Sep 20 22:16:01 2024","interfaces":[{"interface":"back","average":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"min":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":0.91500000000000004,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.68700000000000006},{"interface":"front","average":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"min":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"max":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.59999999999999998}]},{"osd":6,"last update":"Fri Sep 20 22:16:26 2024","interfaces":[{"interface":"back","average":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.89500000000000002,"5min":0.89500000000000002,"15min":0.89500000000000002},"last":0.66900000000000004},{"interface":"front","average":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"min":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"max":{"1min":0.90700000000000003,"5min":0.90700000000000003,"15min":0.90700000000000003},"last":0.55300000000000005}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56699999999999995}]}]},{"osd":3,"up_from":26,"seq":111669149728,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5724,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739428,"statfs":{"total":95995035648,"available":95989174272,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 20 22:16:18 2024","interfaces":[{"interface":"back","average":{"1min":0.39600000000000002,"5min":0.40699999999999997,"15min":0.40899999999999997},"min":{"1min":0.19600000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":0.58099999999999996,"5min":0.59699999999999998,"15min":0.59699999999999998},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.432,"5min":0.438,"15min":0.439},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.41199999999999998}]},{"osd":1,"last update":"Fri Sep 20 22:16:18 2024","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":0.45700000000000002,"15min":0.45200000000000001},"min":{"1min":0.29599999999999999,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.82899999999999996,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.498},{"interface":"front","average":{"1min":0.45500000000000002,"5min":0.45200000000000001,"15min":0.45200000000000001},"min":{"1min":0.312,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.434}]},{"osd":2,"last update":"Fri Sep 20 22:16:18 2024","interfaces":[{"interface":"back","average":{"1min":0.48599999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.76900000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.49399999999999999,"5min":0.47199999999999998,"15min":0.46800000000000003},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.98099999999999998,"5min":0.98099999999999998,"15min":0.98099999999999998},"last":0.39400000000000002}]},{"osd":4,"last update":"Fri Sep 20 22:15:38 2024","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.627,"15min":0.627},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":0.86399999999999999,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.54300000000000004},{"interface":"front","average":{"1min":1.4019999999999999,"5min":1.4019999999999999,"15min":1.4019999999999999},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":17.405999999999999,"5min":17.405999999999999,"15min":17.405999999999999},"last":0.60699999999999998}]},{"osd":5,"last update":"Fri Sep 20 22:16:01 2024","interfaces":[{"interface":"back","average":{"1min":0.65900000000000003,"5min":0.65900000000000003,"15min":0.65900000000000003},"min":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":0.88600000000000001,"5min":0.88600000000000001,"15min":0.88600000000000001},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.63,"5min":0.63,"15min":0.63},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.51400000000000001}]},{"osd":6,"last update":"Fri Sep 20 22:16:24 2024","interfaces":[{"interface":"back","average":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"min":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"max":{"1min":0.872,"5min":0.872,"15min":0.872},"last":0.58699999999999997},{"interface":"front","average":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"min":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"max":{"1min":0.91000000000000003,"5min":0.91000000000000003,"15min":0.91000000000000003},"last":0.625}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.59599999999999997}]}]},{"osd":4,"up_from":31,"seq":133143986203,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5784,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739368,"statfs":{"total":95995035648,"available":95989112832,"internally_reserved":0,"allocated":606208,"data_stored":244968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 20 22:15:39 2024","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.502}]},{"osd":1,"last update":"Fri Sep 20 22:15:39 2024","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.626},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.42299999999999999}]},{"osd":2,"last update":"Fri Sep 20 22:15:39 2024","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.57099999999999995}]},{"osd":3,"last update":"Fri Sep 20 22:15:39 2024","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.55600000000000005}]},{"osd":5,"last update":"Fri Sep 20 22:16:01 2024","interfaces":[{"interface":"back","average":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.439},{"interface":"front","average":{"1min":0.437,"5min":0.437,"15min":0.437},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.65100000000000002}]},{"osd":6,"last update":"Fri Sep 20 22:16:26 2024","interfaces":[{"interface":"back","average":{"1min":0.47799999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.59899999999999998},{"interface":"front","average":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.84299999999999997,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.60999999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.63500000000000001}]}]},{"osd":5,"up_from":39,"seq":167503724568,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5720,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739432,"statfs":{"total":95995035648,"available":95989178368,"internally_reserved":0,"allocated":606208,"data_stored":244910,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 20 22:16:01 2024","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.89500000000000002,"5min":0.89500000000000002,"15min":0.89500000000000002},"last":0.622},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.876,"5min":0.876,"15min":0.876},"last":0.67900000000000005}]},{"osd":1,"last update":"Fri Sep 20 22:16:01 2024","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.86099999999999999,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.84499999999999997,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.64600000000000002}]},{"osd":2,"last update":"Fri Sep 20 22:16:01 2024","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.70799999999999996},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.94099999999999995,"5min":0.94099999999999995,"15min":0.94099999999999995},"last":0.68799999999999994}]},{"osd":3,"last update":"Fri Sep 20 22:16:01 2024","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.66600000000000004},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.92000000000000004,"5min":0.92000000000000004,"15min":0.92000000000000004},"last":0.66100000000000003}]},{"osd":4,"last update":"Fri Sep 20 22:16:01 2024","interfaces":[{"interface":"back","average":{"1min":0.371,"5min":0.371,"15min":0.371},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"last":0.48499999999999999}]},{"osd":6,"last update":"Fri Sep 20 22:16:27 2024","interfaces":[{"interface":"back","average":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.65900000000000003,"5min":0.65900000000000003,"15min":0.65900000000000003},"last":0.442},{"interface":"front","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.52500000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.55800000000000005}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-20T22:16:43.195 DEBUG:teuthology.orchestra.run.smithi037:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -- ceph pg dump --format=json 2024-09-20T22:16:43.409 INFO:teuthology.orchestra.run.smithi037.stderr:Inferring config /var/lib/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/mon.smithi037/config 2024-09-20T22:16:44.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:16:43 smithi194 ceph-mon[25957]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T22:16:44.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:16:43 smithi194 ceph-mon[25957]: from='client.14448 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-20T22:16:44.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:16:43 smithi037 ceph-mon[23718]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T22:16:44.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:16:43 smithi037 ceph-mon[23718]: from='client.14448 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-20T22:16:45.194 INFO:teuthology.orchestra.run.smithi037.stdout: 2024-09-20T22:16:45.195 INFO:teuthology.orchestra.run.smithi037.stderr:dumped all 2024-09-20T22:16:45.875 INFO:teuthology.orchestra.run.smithi037.stdout:{"pg_ready":true,"pg_map":{"version":200,"stamp":"2024-09-20T22:16:44.162211+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48928,"kb_used_data":6304,"kb_used_omap":0,"kb_used_meta":42560,"kb_avail":749912288,"statfs":{"total":767960285184,"available":767910182912,"internally_reserved":0,"allocated":6455296,"data_stored":3546884,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43581440},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002190"},"pg_stats":[{"pgid":"1.0","version":"22'117","reported_seq":226,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-20T22:15:51.682016+0000","last_change":"2024-09-20T22:15:51.682016+0000","last_active":"2024-09-20T22:15:51.682016+0000","last_peered":"2024-09-20T22:15:51.682016+0000","last_clean":"2024-09-20T22:15:51.682016+0000","last_became_active":"2024-09-20T22:15:51.681567+0000","last_became_peered":"2024-09-20T22:15:51.681567+0000","last_unstale":"2024-09-20T22:15:51.682016+0000","last_undegraded":"2024-09-20T22:15:51.682016+0000","last_fullsized":"2024-09-20T22:15:51.682016+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-20T22:13:50.793780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-20T22:13:50.793780+0000","last_clean_scrub_stamp":"2024-09-20T22:13:50.793780+0000","objects_scrubbed":0,"log_size":117,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-21T23:07:56.203597+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3727360,"data_stored":3702496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201294,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6048,"kb_used_data":1112,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739104,"statfs":{"total":95995035648,"available":95988842496,"internally_reserved":0,"allocated":1138688,"data_stored":773838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.63300000000000001}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.55900000000000005}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.68000000000000005}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57899999999999996}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54700000000000004}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.443}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.61599999999999999}]}]},{"osd":6,"up_from":47,"seq":201863462930,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6168,"kb_used_data":1104,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93738984,"statfs":{"total":95995035648,"available":95988719616,"internally_reserved":0,"allocated":1130496,"data_stored":773308,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 20 22:16:24 2024","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.505,"15min":0.505},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.38},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.60199999999999998}]},{"osd":1,"last update":"Fri Sep 20 22:16:24 2024","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.47199999999999998},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.45300000000000001}]},{"osd":2,"last update":"Fri Sep 20 22:16:24 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.86399999999999999,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.98499999999999999,"5min":0.98499999999999999,"15min":0.98499999999999999},"last":0.52200000000000002}]},{"osd":3,"last update":"Fri Sep 20 22:16:24 2024","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.495},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.57999999999999996}]},{"osd":4,"last update":"Fri Sep 20 22:16:24 2024","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"last":0.42399999999999999},{"interface":"front","average":{"1min":0.48999999999999999,"5min":0.48999999999999999,"15min":0.48999999999999999},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"last":0.53400000000000003}]},{"osd":5,"last update":"Fri Sep 20 22:16:24 2024","interfaces":[{"interface":"back","average":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"last":0.622},{"interface":"front","average":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.58799999999999997}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56000000000000005}]}]},{"osd":1,"up_from":14,"seq":60129542187,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6364,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738788,"statfs":{"total":95995035648,"available":95988518912,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 20 22:16:24 2024","interfaces":[{"interface":"back","average":{"1min":0.36899999999999999,"5min":0.34000000000000002,"15min":0.33000000000000002},"min":{"1min":0.25900000000000001,"5min":0.222,"15min":0.222},"max":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.40000000000000002,"5min":0.35899999999999999,"15min":0.34100000000000003},"min":{"1min":0.28100000000000003,"5min":0.221,"15min":0.221},"max":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"last":0.34300000000000003}]},{"osd":2,"last update":"Fri Sep 20 22:15:55 2024","interfaces":[{"interface":"back","average":{"1min":0.39700000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"min":{"1min":0.28699999999999998,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.65100000000000002,"5min":0.746,"15min":0.746},"last":0.35999999999999999},{"interface":"front","average":{"1min":0.44600000000000001,"5min":0.41599999999999998,"15min":0.41099999999999998},"min":{"1min":0.34999999999999998,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":0.40500000000000003}]},{"osd":3,"last update":"Fri Sep 20 22:16:24 2024","interfaces":[{"interface":"back","average":{"1min":0.46700000000000003,"5min":0.47199999999999998,"15min":0.47299999999999998},"min":{"1min":0.33600000000000002,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.501,"5min":0.495,"15min":0.49399999999999999},"min":{"1min":0.33700000000000002,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.53100000000000003}]},{"osd":4,"last update":"Fri Sep 20 22:15:39 2024","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":0.90200000000000002,"5min":0.90200000000000002,"15min":0.90200000000000002},"last":0.52000000000000002}]},{"osd":5,"last update":"Fri Sep 20 22:16:01 2024","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.78100000000000003,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.59199999999999997},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.46899999999999997}]},{"osd":6,"last update":"Fri Sep 20 22:16:24 2024","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":0.624,"15min":0.624},"min":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"max":{"1min":0.84499999999999997,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.501},{"interface":"front","average":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"min":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"max":{"1min":0.84799999999999998,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.61099999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56799999999999995}]}]},{"osd":0,"up_from":9,"seq":38654705714,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6884,"kb_used_data":1116,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738268,"statfs":{"total":95995035648,"available":95987986432,"internally_reserved":0,"allocated":1142784,"data_stored":774161,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Sep 20 22:16:28 2024","interfaces":[{"interface":"back","average":{"1min":0.40200000000000002,"5min":0.34799999999999998,"15min":0.32000000000000001},"min":{"1min":0.248,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.58199999999999996,"5min":0.627,"15min":0.627},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.40400000000000003,"5min":0.34599999999999997,"15min":0.317},"min":{"1min":0.26500000000000001,"5min":0.224,"15min":0.224},"max":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"last":0.498}]},{"osd":2,"last update":"Fri Sep 20 22:15:54 2024","interfaces":[{"interface":"back","average":{"1min":0.45900000000000002,"5min":0.38600000000000001,"15min":0.374},"min":{"1min":0.29899999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"last":0.40600000000000003},{"interface":"front","average":{"1min":0.45000000000000001,"5min":0.38200000000000001,"15min":0.37},"min":{"1min":0.33200000000000002,"5min":0.251,"15min":0.251},"max":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.30399999999999999}]},{"osd":3,"last update":"Fri Sep 20 22:16:24 2024","interfaces":[{"interface":"back","average":{"1min":0.435,"5min":0.46200000000000002,"15min":0.46600000000000003},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.57399999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.46899999999999997,"5min":0.49299999999999999,"15min":0.497},"min":{"1min":0.35399999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.35699999999999998}]},{"osd":4,"last update":"Fri Sep 20 22:16:38 2024","interfaces":[{"interface":"back","average":{"1min":0.66500000000000004,"5min":0.629,"15min":0.623},"min":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"max":{"1min":0.92100000000000004,"5min":0.99399999999999999,"15min":0.99399999999999999},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.66900000000000004,"5min":0.63100000000000001,"15min":0.625},"min":{"1min":0.42499999999999999,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.93200000000000005,"5min":0.98299999999999998,"15min":0.98299999999999998},"last":0.73499999999999999}]},{"osd":5,"last update":"Fri Sep 20 22:16:00 2024","interfaces":[{"interface":"back","average":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"min":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"max":{"1min":0.90500000000000003,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.64500000000000002},{"interface":"front","average":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"min":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":0.91000000000000003,"5min":0.91000000000000003,"15min":0.91000000000000003},"last":0.65400000000000003}]},{"osd":6,"last update":"Fri Sep 20 22:16:24 2024","interfaces":[{"interface":"back","average":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.78400000000000003},{"interface":"front","average":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"min":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.61499999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.60799999999999998}]}]},{"osd":2,"up_from":19,"seq":81604378662,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6236,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5632,"kb_avail":93738916,"statfs":{"total":95995035648,"available":95988649984,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5767168},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 20 22:15:52 2024","interfaces":[{"interface":"back","average":{"1min":0.45700000000000002,"5min":0.39000000000000001,"15min":0.379},"min":{"1min":0.29699999999999999,"5min":0.216,"15min":0.216},"max":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.23300000000000001},{"interface":"front","average":{"1min":0.46200000000000002,"5min":0.41499999999999998,"15min":0.40699999999999997},"min":{"1min":0.27700000000000002,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.32900000000000001}]},{"osd":1,"last update":"Fri Sep 20 22:15:52 2024","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":0.38600000000000001,"15min":0.36799999999999999},"min":{"1min":0.26400000000000001,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.27000000000000002},{"interface":"front","average":{"1min":0.48899999999999999,"5min":0.41199999999999998,"15min":0.39900000000000002},"min":{"1min":0.28399999999999997,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.39800000000000002}]},{"osd":3,"last update":"Fri Sep 20 22:16:20 2024","interfaces":[{"interface":"back","average":{"1min":0.46899999999999997,"5min":0.48399999999999999,"15min":0.48599999999999999},"min":{"1min":0.317,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.35699999999999998},{"interface":"front","average":{"1min":0.42999999999999999,"5min":0.46500000000000002,"15min":0.47099999999999997},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"last":0.29299999999999998}]},{"osd":4,"last update":"Fri Sep 20 22:15:40 2024","interfaces":[{"interface":"back","average":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"min":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":1.121,"5min":1.121,"15min":1.121},"last":0.61799999999999999},{"interface":"front","average":{"1min":0.69399999999999995,"5min":0.69399999999999995,"15min":0.69399999999999995},"min":{"1min":0.47099999999999997,"5min":0.47099999999999997,"15min":0.47099999999999997},"max":{"1min":1.1399999999999999,"5min":1.1399999999999999,"15min":1.1399999999999999},"last":0.64800000000000002}]},{"osd":5,"last update":"Fri Sep 20 22:16:01 2024","interfaces":[{"interface":"back","average":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"min":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":0.91500000000000004,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.68700000000000006},{"interface":"front","average":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"min":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"max":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.59999999999999998}]},{"osd":6,"last update":"Fri Sep 20 22:16:26 2024","interfaces":[{"interface":"back","average":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.89500000000000002,"5min":0.89500000000000002,"15min":0.89500000000000002},"last":0.66900000000000004},{"interface":"front","average":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"min":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"max":{"1min":0.90700000000000003,"5min":0.90700000000000003,"15min":0.90700000000000003},"last":0.55300000000000005}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56699999999999995}]}]},{"osd":3,"up_from":26,"seq":111669149729,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5724,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739428,"statfs":{"total":95995035648,"available":95989174272,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 20 22:16:18 2024","interfaces":[{"interface":"back","average":{"1min":0.39600000000000002,"5min":0.40699999999999997,"15min":0.40899999999999997},"min":{"1min":0.19600000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":0.58099999999999996,"5min":0.59699999999999998,"15min":0.59699999999999998},"last":0.36099999999999999},{"interface":"front","average":{"1min":0.432,"5min":0.438,"15min":0.439},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.46200000000000002}]},{"osd":1,"last update":"Fri Sep 20 22:16:18 2024","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":0.45700000000000002,"15min":0.45200000000000001},"min":{"1min":0.29599999999999999,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.82899999999999996,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.41199999999999998},{"interface":"front","average":{"1min":0.45500000000000002,"5min":0.45200000000000001,"15min":0.45200000000000001},"min":{"1min":0.312,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.40600000000000003}]},{"osd":2,"last update":"Fri Sep 20 22:16:18 2024","interfaces":[{"interface":"back","average":{"1min":0.48599999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.76900000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.44600000000000001},{"interface":"front","average":{"1min":0.49399999999999999,"5min":0.47199999999999998,"15min":0.46800000000000003},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.98099999999999998,"5min":0.98099999999999998,"15min":0.98099999999999998},"last":0.437}]},{"osd":4,"last update":"Fri Sep 20 22:16:39 2024","interfaces":[{"interface":"back","average":{"1min":0.67100000000000004,"5min":0.63500000000000001,"15min":0.629},"min":{"1min":0.49199999999999999,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":0.81100000000000005,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.69199999999999995,"5min":1.26,"15min":1.3540000000000001},"min":{"1min":0.55800000000000005,"5min":0.44,"15min":0.44},"max":{"1min":0.85799999999999998,"5min":17.405999999999999,"15min":17.405999999999999},"last":0.47999999999999998}]},{"osd":5,"last update":"Fri Sep 20 22:16:01 2024","interfaces":[{"interface":"back","average":{"1min":0.65900000000000003,"5min":0.65900000000000003,"15min":0.65900000000000003},"min":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":0.88600000000000001,"5min":0.88600000000000001,"15min":0.88600000000000001},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.63,"5min":0.63,"15min":0.63},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.53900000000000003}]},{"osd":6,"last update":"Fri Sep 20 22:16:24 2024","interfaces":[{"interface":"back","average":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"min":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"max":{"1min":0.872,"5min":0.872,"15min":0.872},"last":0.64900000000000002},{"interface":"front","average":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"min":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"max":{"1min":0.91000000000000003,"5min":0.91000000000000003,"15min":0.91000000000000003},"last":0.48699999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.52000000000000002}]}]},{"osd":4,"up_from":31,"seq":133143986204,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5784,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739368,"statfs":{"total":95995035648,"available":95989112832,"internally_reserved":0,"allocated":606208,"data_stored":244968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 20 22:16:39 2024","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.57099999999999995,"15min":0.57699999999999996},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.72699999999999998,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.53900000000000003},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.60199999999999998,"15min":0.60599999999999998},"min":{"1min":0.44600000000000001,"5min":0.435,"15min":0.435},"max":{"1min":0.73999999999999999,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.56799999999999995}]},{"osd":1,"last update":"Fri Sep 20 22:16:39 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.56999999999999995,"15min":0.57399999999999995},"min":{"1min":0.41999999999999998,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.70499999999999996,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.41699999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.58199999999999996}]},{"osd":2,"last update":"Fri Sep 20 22:16:39 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.749,"5min":0.749,"15min":0.749},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.58299999999999996,"15min":0.58799999999999997},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.748,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.64200000000000002}]},{"osd":3,"last update":"Fri Sep 20 22:16:39 2024","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.55600000000000005,"15min":0.56000000000000005},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.499},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.57699999999999996,"15min":0.58099999999999996},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.73699999999999999,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.46899999999999997}]},{"osd":5,"last update":"Fri Sep 20 22:16:01 2024","interfaces":[{"interface":"back","average":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.61399999999999999},{"interface":"front","average":{"1min":0.437,"5min":0.437,"15min":0.437},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.55600000000000005}]},{"osd":6,"last update":"Fri Sep 20 22:16:26 2024","interfaces":[{"interface":"back","average":{"1min":0.47799999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.84299999999999997,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.66200000000000003}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.68300000000000005}]}]},{"osd":5,"up_from":39,"seq":167503724568,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5720,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739432,"statfs":{"total":95995035648,"available":95989178368,"internally_reserved":0,"allocated":606208,"data_stored":244910,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 20 22:16:01 2024","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.89500000000000002,"5min":0.89500000000000002,"15min":0.89500000000000002},"last":0.622},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.876,"5min":0.876,"15min":0.876},"last":0.67900000000000005}]},{"osd":1,"last update":"Fri Sep 20 22:16:01 2024","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.86099999999999999,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.84499999999999997,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.64600000000000002}]},{"osd":2,"last update":"Fri Sep 20 22:16:01 2024","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.70799999999999996},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.94099999999999995,"5min":0.94099999999999995,"15min":0.94099999999999995},"last":0.68799999999999994}]},{"osd":3,"last update":"Fri Sep 20 22:16:01 2024","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.66600000000000004},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.92000000000000004,"5min":0.92000000000000004,"15min":0.92000000000000004},"last":0.66100000000000003}]},{"osd":4,"last update":"Fri Sep 20 22:16:01 2024","interfaces":[{"interface":"back","average":{"1min":0.371,"5min":0.371,"15min":0.371},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"last":0.48499999999999999}]},{"osd":6,"last update":"Fri Sep 20 22:16:27 2024","interfaces":[{"interface":"back","average":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.65900000000000003,"5min":0.65900000000000003,"15min":0.65900000000000003},"last":0.442},{"interface":"front","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.52500000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.55800000000000005}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-20T22:16:45.876 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-09-20T22:16:45.876 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-09-20T22:16:45.876 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-09-20T22:16:45.876 DEBUG:teuthology.orchestra.run.smithi037:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -- ceph health --format=json 2024-09-20T22:16:46.091 INFO:teuthology.orchestra.run.smithi037.stderr:Inferring config /var/lib/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/mon.smithi037/config 2024-09-20T22:16:46.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:16:45 smithi194 ceph-mon[25957]: pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T22:16:46.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:16:45 smithi037 ceph-mon[23718]: pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T22:16:47.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:16:46 smithi194 ceph-mon[25957]: from='client.14452 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-20T22:16:47.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:16:46 smithi037 ceph-mon[23718]: from='client.14452 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-20T22:16:48.017 INFO:teuthology.orchestra.run.smithi037.stdout: 2024-09-20T22:16:48.017 INFO:teuthology.orchestra.run.smithi037.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-09-20T22:16:48.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:16:47 smithi194 ceph-mon[25957]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T22:16:48.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:16:47 smithi037 ceph-mon[23718]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T22:16:48.606 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-09-20T22:16:48.607 INFO:tasks.cephadm:Setup complete, yielding 2024-09-20T22:16:48.607 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-20T22:16:48.616 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi037.front.sepia.ceph.com 2024-09-20T22:16:48.616 DEBUG:teuthology.orchestra.run.smithi037:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -- bash -c 'ceph orch status' 2024-09-20T22:16:48.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:16:48 smithi037 ceph-mon[23718]: from='client.? 172.21.15.37:0/1765782809' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-20T22:16:49.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:16:48 smithi194 ceph-mon[25957]: from='client.? 172.21.15.37:0/1765782809' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-20T22:16:49.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:16:49 smithi037 ceph-mon[23718]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T22:16:49.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:16:49 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:16:49.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:16:49 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:16:49.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:16:49 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:16:50.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:16:49 smithi194 ceph-mon[25957]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T22:16:50.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:16:49 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:16:50.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:16:49 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:16:50.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:16:49 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:16:50.643 INFO:teuthology.orchestra.run.smithi037.stdout:Backend: cephadm 2024-09-20T22:16:50.643 INFO:teuthology.orchestra.run.smithi037.stdout:Available: Yes 2024-09-20T22:16:50.643 INFO:teuthology.orchestra.run.smithi037.stdout:Paused: No 2024-09-20T22:16:51.285 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:16:51 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:16:51.286 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:16:51 smithi037 ceph-mon[23718]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T22:16:51.286 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:16:51 smithi037 ceph-mon[23718]: from='client.14460 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:16:51.312 DEBUG:teuthology.orchestra.run.smithi037:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -- bash -c 'ceph orch ps' 2024-09-20T22:16:51.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:16:51 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:16:51.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:16:51 smithi194 ceph-mon[25957]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T22:16:51.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:16:51 smithi194 ceph-mon[25957]: from='client.14460 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:16:53.263 INFO:teuthology.orchestra.run.smithi037.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T22:16:53.263 INFO:teuthology.orchestra.run.smithi037.stdout:alertmanager.smithi037 smithi037 *:9093,9094 running (4m) 2m ago 5m 15.5M - ba2b418f427c bb407b78f0c9 2024-09-20T22:16:53.263 INFO:teuthology.orchestra.run.smithi037.stdout:crash.smithi037 smithi037 running (5m) 2m ago 5m 6400k - 17.2.0 e1d6a67b021e 88301022bdc8 2024-09-20T22:16:53.264 INFO:teuthology.orchestra.run.smithi037.stdout:crash.smithi194 smithi194 running (5m) 66s ago 5m 6405k - 17.2.0 e1d6a67b021e b553601d7ca5 2024-09-20T22:16:53.264 INFO:teuthology.orchestra.run.smithi037.stdout:grafana.smithi037 smithi037 *:3000 running (3m) 2m ago 5m 43.0M - 8.3.5 dad864ee21e9 3ccb7bb6f7ae 2024-09-20T22:16:53.264 INFO:teuthology.orchestra.run.smithi037.stdout:mgr.smithi037.lykblh smithi037 *:9283 running (6m) 2m ago 6m 448M - 17.2.0 e1d6a67b021e 3ef1f99a6ba9 2024-09-20T22:16:53.264 INFO:teuthology.orchestra.run.smithi037.stdout:mgr.smithi194.bkcglt smithi194 *:8443,9283 running (5m) 66s ago 5m 412M - 17.2.0 e1d6a67b021e 7cc5497632b6 2024-09-20T22:16:53.264 INFO:teuthology.orchestra.run.smithi037.stdout:mon.smithi037 smithi037 running (7m) 2m ago 7m 41.7M 2048M 17.2.0 e1d6a67b021e 71f591122786 2024-09-20T22:16:53.264 INFO:teuthology.orchestra.run.smithi037.stdout:mon.smithi194 smithi194 running (4m) 66s ago 4m 39.5M 2048M 17.2.0 e1d6a67b021e 0b8814ae6c66 2024-09-20T22:16:53.264 INFO:teuthology.orchestra.run.smithi037.stdout:node-exporter.smithi037 smithi037 *:9100 running (5m) 2m ago 5m 16.5M - 1dbe0e931976 1f92060f034e 2024-09-20T22:16:53.264 INFO:teuthology.orchestra.run.smithi037.stdout:node-exporter.smithi194 smithi194 *:9100 running (4m) 66s ago 4m 17.5M - 1dbe0e931976 3a80a2ef6678 2024-09-20T22:16:53.264 INFO:teuthology.orchestra.run.smithi037.stdout:osd.0 smithi037 running (4m) 2m ago 4m 34.8M 3217M 17.2.0 e1d6a67b021e e54e84fd5030 2024-09-20T22:16:53.264 INFO:teuthology.orchestra.run.smithi037.stdout:osd.1 smithi037 running (3m) 2m ago 3m 34.3M 3217M 17.2.0 e1d6a67b021e 09647c133032 2024-09-20T22:16:53.264 INFO:teuthology.orchestra.run.smithi037.stdout:osd.2 smithi037 running (3m) 2m ago 3m 34.2M 3217M 17.2.0 e1d6a67b021e 6c15b1175256 2024-09-20T22:16:53.264 INFO:teuthology.orchestra.run.smithi037.stdout:osd.3 smithi037 running (2m) 2m ago 2m 25.4M 3217M 17.2.0 e1d6a67b021e 3a22952c6475 2024-09-20T22:16:53.264 INFO:teuthology.orchestra.run.smithi037.stdout:osd.4 smithi194 running (2m) 66s ago 2m 32.8M 3984M 17.2.0 e1d6a67b021e fa107be8915c 2024-09-20T22:16:53.264 INFO:teuthology.orchestra.run.smithi037.stdout:osd.5 smithi194 running (119s) 66s ago 118s 32.8M 3984M 17.2.0 e1d6a67b021e 290b21cbd131 2024-09-20T22:16:53.264 INFO:teuthology.orchestra.run.smithi037.stdout:osd.6 smithi194 running (95s) 66s ago 95s 33.1M 3984M 17.2.0 e1d6a67b021e 27f0720639ed 2024-09-20T22:16:53.265 INFO:teuthology.orchestra.run.smithi037.stdout:osd.7 smithi194 running (71s) 66s ago 71s 12.0M 3984M 17.2.0 e1d6a67b021e 62d163f4fd92 2024-09-20T22:16:53.265 INFO:teuthology.orchestra.run.smithi037.stdout:prometheus.smithi037 smithi037 *:9095 running (4m) 2m ago 4m 53.2M - 514e6a882f6e 32020a1cc2f4 2024-09-20T22:16:53.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:16:53 smithi194 ceph-mon[25957]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T22:16:53.662 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:16:53 smithi037 ceph-mon[23718]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T22:16:53.903 DEBUG:teuthology.orchestra.run.smithi037:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -- bash -c 'ceph orch ls' 2024-09-20T22:16:54.872 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:16:54 smithi037 ceph-mon[23718]: from='client.14464 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:16:55.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:16:54 smithi194 ceph-mon[25957]: from='client.14464 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:16:55.813 INFO:teuthology.orchestra.run.smithi037.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2024-09-20T22:16:55.813 INFO:teuthology.orchestra.run.smithi037.stdout:alertmanager ?:9093,9094 1/1 2m ago 6m count:1 2024-09-20T22:16:55.813 INFO:teuthology.orchestra.run.smithi037.stdout:crash 2/2 2m ago 6m * 2024-09-20T22:16:55.813 INFO:teuthology.orchestra.run.smithi037.stdout:grafana ?:3000 1/1 2m ago 6m count:1 2024-09-20T22:16:55.813 INFO:teuthology.orchestra.run.smithi037.stdout:mgr 2/2 2m ago 6m count:2 2024-09-20T22:16:55.813 INFO:teuthology.orchestra.run.smithi037.stdout:mon 2/2 2m ago 5m smithi037:172.21.15.37=smithi037;smithi194:172.21.15.194=smithi194;count:2 2024-09-20T22:16:55.813 INFO:teuthology.orchestra.run.smithi037.stdout:node-exporter ?:9100 2/2 2m ago 6m * 2024-09-20T22:16:55.814 INFO:teuthology.orchestra.run.smithi037.stdout:osd 8 2m ago - 2024-09-20T22:16:55.814 INFO:teuthology.orchestra.run.smithi037.stdout:prometheus ?:9095 1/1 2m ago 6m count:1 2024-09-20T22:16:56.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:16:55 smithi194 ceph-mon[25957]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T22:16:56.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:16:55 smithi037 ceph-mon[23718]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T22:16:56.464 DEBUG:teuthology.orchestra.run.smithi037:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -- bash -c 'ceph orch host ls' 2024-09-20T22:16:57.006 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:16:56 smithi037 ceph-mon[23718]: from='client.14468 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:16:57.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:16:56 smithi194 ceph-mon[25957]: from='client.14468 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:16:57.959 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:16:57 smithi037 ceph-mon[23718]: pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T22:16:58.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:16:57 smithi194 ceph-mon[25957]: pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T22:16:58.391 INFO:teuthology.orchestra.run.smithi037.stdout:HOST ADDR LABELS STATUS 2024-09-20T22:16:58.391 INFO:teuthology.orchestra.run.smithi037.stdout:smithi037 172.21.15.37 2024-09-20T22:16:58.392 INFO:teuthology.orchestra.run.smithi037.stdout:smithi194 172.21.15.194 2024-09-20T22:16:58.392 INFO:teuthology.orchestra.run.smithi037.stdout:2 hosts in cluster 2024-09-20T22:16:59.024 DEBUG:teuthology.orchestra.run.smithi037:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -- bash -c 'ceph orch device ls' 2024-09-20T22:16:59.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:16:59 smithi037 ceph-mon[23718]: pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T22:16:59.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:16:59 smithi037 ceph-mon[23718]: from='client.14472 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:17:00.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:16:59 smithi194 ceph-mon[25957]: pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T22:17:00.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:16:59 smithi194 ceph-mon[25957]: from='client.14472 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:17:01.044 INFO:teuthology.orchestra.run.smithi037.stdout:HOST PATH TYPE DEVICE ID SIZE AVAILABLE REJECT REASONS 2024-09-20T22:17:01.044 INFO:teuthology.orchestra.run.smithi037.stdout:smithi037 /dev/nvme0n1 ssd INTEL_SSDPEDMD400G4_CVFT5331005T400BGN 400G LVM detected, locked 2024-09-20T22:17:01.044 INFO:teuthology.orchestra.run.smithi037.stdout:smithi194 /dev/nvme0n1 ssd INTEL_SSDPEDMD400G4_CVFT6240002A400BGN 400G LVM detected, locked 2024-09-20T22:17:01.663 INFO:teuthology.run_tasks:Running task vip.exec... 2024-09-20T22:17:01.680 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi037.front.sepia.ceph.com 2024-09-20T22:17:01.680 DEBUG:teuthology.orchestra.run.smithi037:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'systemctl stop nfs-server' 2024-09-20T22:17:01.708 INFO:teuthology.orchestra.run.smithi037.stderr:+ systemctl stop nfs-server 2024-09-20T22:17:01.717 INFO:tasks.vip:Running commands on role host.b host ubuntu@smithi194.front.sepia.ceph.com 2024-09-20T22:17:01.717 DEBUG:teuthology.orchestra.run.smithi194:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'systemctl stop nfs-server' 2024-09-20T22:17:01.747 INFO:teuthology.orchestra.run.smithi194.stderr:+ systemctl stop nfs-server 2024-09-20T22:17:01.755 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-20T22:17:01.763 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi037.front.sepia.ceph.com 2024-09-20T22:17:01.763 DEBUG:teuthology.orchestra.run.smithi037:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -- bash -c 'ceph fs volume create foofs' 2024-09-20T22:17:01.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:01 smithi037 ceph-mon[23718]: pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T22:17:02.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:01 smithi194 ceph-mon[25957]: pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T22:17:02.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:02 smithi037 ceph-mon[23718]: from='client.14476 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:17:03.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:02 smithi194 ceph-mon[25957]: from='client.14476 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:17:04.412 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:04 smithi037 ceph-mon[23718]: pgmap v209: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T22:17:04.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:04 smithi194 ceph-mon[25957]: pgmap v209: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T22:17:05.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:05 smithi037 ceph-mon[23718]: from='client.14480 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foofs", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:17:05.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:05 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]: dispatch 2024-09-20T22:17:05.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:05 smithi037 ceph-mon[23718]: pgmap v210: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T22:17:05.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:05 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]': finished 2024-09-20T22:17:05.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:05 smithi037 ceph-mon[23718]: osdmap e60: 8 total, 8 up, 8 in 2024-09-20T22:17:05.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:05 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]: dispatch 2024-09-20T22:17:05.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:05 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-mon-smithi037[23714]: 2024-09-20T22:17:05.178+0000 7f46f8f13700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-20T22:17:05.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:05 smithi194 ceph-mon[25957]: from='client.14480 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foofs", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:17:05.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:05 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]: dispatch 2024-09-20T22:17:05.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:05 smithi194 ceph-mon[25957]: pgmap v210: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T22:17:05.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:05 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]': finished 2024-09-20T22:17:05.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:05 smithi194 ceph-mon[25957]: osdmap e60: 8 total, 8 up, 8 in 2024-09-20T22:17:05.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:05 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]: dispatch 2024-09-20T22:17:06.007 INFO:teuthology.run_tasks:Running task cephadm.wait_for_service... 2024-09-20T22:17:06.016 INFO:tasks.cephadm:Waiting for ceph service mds.foofs to start (timeout 300)... 2024-09-20T22:17:06.016 DEBUG:teuthology.orchestra.run.smithi037:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -- ceph orch ls -f json 2024-09-20T22:17:06.383 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:06 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]': finished 2024-09-20T22:17:06.383 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:06 smithi037 ceph-mon[23718]: osdmap e61: 8 total, 8 up, 8 in 2024-09-20T22:17:06.383 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:06 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]: dispatch 2024-09-20T22:17:06.384 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:06 smithi037 ceph-mon[23718]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-20T22:17:06.384 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:06 smithi037 ceph-mon[23718]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-09-20T22:17:06.384 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:06 smithi037 ceph-mon[23718]: osdmap e62: 8 total, 8 up, 8 in 2024-09-20T22:17:06.384 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:06 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]': finished 2024-09-20T22:17:06.384 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:06 smithi037 ceph-mon[23718]: fsmap foofs:0 2024-09-20T22:17:06.384 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:06 smithi037 ceph-mon[23718]: Saving service mds.foofs spec with placement count:2 2024-09-20T22:17:06.384 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:06 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:17:06.384 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:06 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:17:06.384 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:06 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:17:06.384 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:06 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:17:06.384 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:06 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:17:06.384 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:06 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:17:06.384 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:06 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi037.sdtedi", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-20T22:17:06.384 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:06 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi037.sdtedi", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-20T22:17:06.384 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:06 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:17:06.385 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:06 smithi037 ceph-mon[23718]: Deploying daemon mds.foofs.smithi037.sdtedi on smithi037 2024-09-20T22:17:06.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:06 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]': finished 2024-09-20T22:17:06.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:06 smithi194 ceph-mon[25957]: osdmap e61: 8 total, 8 up, 8 in 2024-09-20T22:17:06.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:06 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]: dispatch 2024-09-20T22:17:06.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:06 smithi194 ceph-mon[25957]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-20T22:17:06.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:06 smithi194 ceph-mon[25957]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-09-20T22:17:06.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:06 smithi194 ceph-mon[25957]: osdmap e62: 8 total, 8 up, 8 in 2024-09-20T22:17:06.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:06 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]': finished 2024-09-20T22:17:06.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:06 smithi194 ceph-mon[25957]: fsmap foofs:0 2024-09-20T22:17:06.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:06 smithi194 ceph-mon[25957]: Saving service mds.foofs spec with placement count:2 2024-09-20T22:17:06.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:06 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:17:06.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:06 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:17:06.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:06 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:17:06.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:06 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:17:06.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:06 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:17:06.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:06 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:17:06.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:06 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi037.sdtedi", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-20T22:17:06.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:06 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi037.sdtedi", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-20T22:17:06.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:06 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:17:06.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:06 smithi194 ceph-mon[25957]: Deploying daemon mds.foofs.smithi037.sdtedi on smithi037 2024-09-20T22:17:07.412 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:07 smithi037 ceph-mon[23718]: pgmap v214: 65 pgs: 11 creating+peering, 5 active+clean, 49 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T22:17:07.412 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:07 smithi037 ceph-mon[23718]: osdmap e63: 8 total, 8 up, 8 in 2024-09-20T22:17:07.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:07 smithi194 ceph-mon[25957]: pgmap v214: 65 pgs: 11 creating+peering, 5 active+clean, 49 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T22:17:07.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:07 smithi194 ceph-mon[25957]: osdmap e63: 8 total, 8 up, 8 in 2024-09-20T22:17:08.523 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:08 smithi037 ceph-mon[23718]: osdmap e64: 8 total, 8 up, 8 in 2024-09-20T22:17:08.523 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:08 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:17:08.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:08 smithi194 ceph-mon[25957]: osdmap e64: 8 total, 8 up, 8 in 2024-09-20T22:17:08.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:08 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:17:09.183 INFO:teuthology.orchestra.run.smithi037.stdout: 2024-09-20T22:17:09.183 INFO:teuthology.orchestra.run.smithi037.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-20T22:10:35.834131Z", "last_refresh": "2024-09-20T22:14:24.829022Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-20T22:10:29.668492Z", "last_refresh": "2024-09-20T22:14:24.829088Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-09-20T22:10:32.691543Z", "last_refresh": "2024-09-20T22:14:24.829152Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-09-20T22:17:05.197680Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-20T22:17:05.191626Z", "running": 0, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-20T22:10:28.369386Z", "last_refresh": "2024-09-20T22:14:24.828948Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi037:172.21.15.37=smithi037", "smithi194:172.21.15.194=smithi194"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-20T22:11:19.388507Z", "last_refresh": "2024-09-20T22:14:24.828805Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-20T22:10:34.364687Z", "last_refresh": "2024-09-20T22:14:24.829215Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-09-20T22:14:24.829341Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-20T22:10:31.440666Z", "last_refresh": "2024-09-20T22:14:24.829278Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-20T22:17:09.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:09 smithi194 ceph-mon[25957]: pgmap v217: 65 pgs: 3 creating+activating, 11 creating+peering, 46 active+clean, 5 unknown; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-09-20T22:17:09.662 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:09 smithi037 ceph-mon[23718]: pgmap v217: 65 pgs: 3 creating+activating, 11 creating+peering, 46 active+clean, 5 unknown; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-09-20T22:17:09.836 INFO:tasks.cephadm:mds.foofs has 0/2 2024-09-20T22:17:10.412 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:10 smithi037 ceph-mon[23718]: from='client.14484 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-20T22:17:10.412 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:10 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:17:10.412 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:10 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi194.lkroso", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-20T22:17:10.412 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:10 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi194.lkroso", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-20T22:17:10.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:10 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:17:10.551 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:10 smithi194 ceph-mon[25957]: from='client.14484 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-20T22:17:10.551 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:10 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:17:10.551 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:10 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi194.lkroso", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-20T22:17:10.551 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:10 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi194.lkroso", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-20T22:17:10.551 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:10 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:17:10.837 DEBUG:teuthology.orchestra.run.smithi037:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -- ceph orch ls -f json 2024-09-20T22:17:11.411 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:11 smithi037 ceph-mon[23718]: Deploying daemon mds.foofs.smithi194.lkroso on smithi194 2024-09-20T22:17:11.411 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:11 smithi037 ceph-mon[23718]: pgmap v218: 65 pgs: 3 creating+activating, 62 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-09-20T22:17:11.411 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:11 smithi037 ceph-mon[23718]: mds.? [v2:172.21.15.37:6834/2142770746,v1:172.21.15.37:6835/2142770746] up:boot 2024-09-20T22:17:11.411 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:11 smithi037 ceph-mon[23718]: daemon mds.foofs.smithi037.sdtedi assigned to filesystem foofs as rank 0 (now has 1 ranks) 2024-09-20T22:17:11.411 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:11 smithi037 ceph-mon[23718]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-20T22:17:11.411 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:11 smithi037 ceph-mon[23718]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-09-20T22:17:11.411 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:11 smithi037 ceph-mon[23718]: Cluster is now healthy 2024-09-20T22:17:11.411 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:11 smithi037 ceph-mon[23718]: fsmap foofs:0 1 up:standby 2024-09-20T22:17:11.411 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:11 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi037.sdtedi"}]: dispatch 2024-09-20T22:17:11.411 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:11 smithi037 ceph-mon[23718]: fsmap foofs:1 {0=foofs.smithi037.sdtedi=up:creating} 2024-09-20T22:17:11.412 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:11 smithi037 ceph-mon[23718]: daemon mds.foofs.smithi037.sdtedi is now active in filesystem foofs as rank 0 2024-09-20T22:17:11.556 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:11 smithi194 ceph-mon[25957]: Deploying daemon mds.foofs.smithi194.lkroso on smithi194 2024-09-20T22:17:11.556 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:11 smithi194 ceph-mon[25957]: pgmap v218: 65 pgs: 3 creating+activating, 62 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-09-20T22:17:11.556 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:11 smithi194 ceph-mon[25957]: mds.? [v2:172.21.15.37:6834/2142770746,v1:172.21.15.37:6835/2142770746] up:boot 2024-09-20T22:17:11.557 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:11 smithi194 ceph-mon[25957]: daemon mds.foofs.smithi037.sdtedi assigned to filesystem foofs as rank 0 (now has 1 ranks) 2024-09-20T22:17:11.557 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:11 smithi194 ceph-mon[25957]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-20T22:17:11.557 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:11 smithi194 ceph-mon[25957]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-09-20T22:17:11.557 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:11 smithi194 ceph-mon[25957]: Cluster is now healthy 2024-09-20T22:17:11.557 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:11 smithi194 ceph-mon[25957]: fsmap foofs:0 1 up:standby 2024-09-20T22:17:11.557 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:11 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi037.sdtedi"}]: dispatch 2024-09-20T22:17:11.557 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:11 smithi194 ceph-mon[25957]: fsmap foofs:1 {0=foofs.smithi037.sdtedi=up:creating} 2024-09-20T22:17:11.557 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:11 smithi194 ceph-mon[25957]: daemon mds.foofs.smithi037.sdtedi is now active in filesystem foofs as rank 0 2024-09-20T22:17:12.390 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:12 smithi037 ceph-mon[23718]: mds.? [v2:172.21.15.37:6834/2142770746,v1:172.21.15.37:6835/2142770746] up:active 2024-09-20T22:17:12.390 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:12 smithi037 ceph-mon[23718]: fsmap foofs:1 {0=foofs.smithi037.sdtedi=up:active} 2024-09-20T22:17:12.479 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:12 smithi194 ceph-mon[25957]: mds.? [v2:172.21.15.37:6834/2142770746,v1:172.21.15.37:6835/2142770746] up:active 2024-09-20T22:17:12.479 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:12 smithi194 ceph-mon[25957]: fsmap foofs:1 {0=foofs.smithi037.sdtedi=up:active} 2024-09-20T22:17:12.830 INFO:teuthology.orchestra.run.smithi037.stdout: 2024-09-20T22:17:12.830 INFO:teuthology.orchestra.run.smithi037.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-20T22:10:35.834131Z", "last_refresh": "2024-09-20T22:14:24.829022Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-20T22:10:29.668492Z", "last_refresh": "2024-09-20T22:14:24.829088Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-09-20T22:10:32.691543Z", "last_refresh": "2024-09-20T22:14:24.829152Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-09-20T22:17:05.197680Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-20T22:17:05.191626Z", "running": 0, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-20T22:10:28.369386Z", "last_refresh": "2024-09-20T22:14:24.828948Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi037:172.21.15.37=smithi037", "smithi194:172.21.15.194=smithi194"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-20T22:11:19.388507Z", "last_refresh": "2024-09-20T22:14:24.828805Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-20T22:10:34.364687Z", "last_refresh": "2024-09-20T22:14:24.829215Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-09-20T22:14:24.829341Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-20T22:10:31.440666Z", "last_refresh": "2024-09-20T22:14:24.829278Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-20T22:17:13.512 INFO:tasks.cephadm:mds.foofs has 0/2 2024-09-20T22:17:13.794 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:13 smithi194 ceph-mon[25957]: pgmap v219: 65 pgs: 65 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 2 op/s 2024-09-20T22:17:13.794 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:13 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:17:13.795 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:13 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:17:13.795 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:13 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:17:13.795 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:13 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:17:13.795 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:13 smithi194 ceph-mon[25957]: from='client.14496 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-20T22:17:13.795 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:13 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:17:13.795 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:13 smithi194 ceph-mon[25957]: mds.? [v2:172.21.15.194:6832/1702667629,v1:172.21.15.194:6833/1702667629] up:boot 2024-09-20T22:17:13.795 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:13 smithi194 ceph-mon[25957]: fsmap foofs:1 {0=foofs.smithi037.sdtedi=up:active} 1 up:standby 2024-09-20T22:17:13.795 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:13 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi194.lkroso"}]: dispatch 2024-09-20T22:17:13.795 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:13 smithi194 ceph-mon[25957]: fsmap foofs:1 {0=foofs.smithi037.sdtedi=up:active} 1 up:standby 2024-09-20T22:17:13.818 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:13 smithi037 ceph-mon[23718]: pgmap v219: 65 pgs: 65 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 2 op/s 2024-09-20T22:17:13.818 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:13 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:17:13.818 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:13 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:17:13.818 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:13 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:17:13.819 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:13 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:17:13.819 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:13 smithi037 ceph-mon[23718]: from='client.14496 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-20T22:17:13.819 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:13 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:17:13.819 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:13 smithi037 ceph-mon[23718]: mds.? [v2:172.21.15.194:6832/1702667629,v1:172.21.15.194:6833/1702667629] up:boot 2024-09-20T22:17:13.819 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:13 smithi037 ceph-mon[23718]: fsmap foofs:1 {0=foofs.smithi037.sdtedi=up:active} 1 up:standby 2024-09-20T22:17:13.819 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:13 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi194.lkroso"}]: dispatch 2024-09-20T22:17:13.819 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:13 smithi037 ceph-mon[23718]: fsmap foofs:1 {0=foofs.smithi037.sdtedi=up:active} 1 up:standby 2024-09-20T22:17:14.512 DEBUG:teuthology.orchestra.run.smithi037:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -- ceph orch ls -f json 2024-09-20T22:17:15.662 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:15 smithi037 ceph-mon[23718]: pgmap v220: 65 pgs: 65 active+clean; 577 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1023 B/s wr, 3 op/s 2024-09-20T22:17:15.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:15 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:17:15.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:15 smithi194 ceph-mon[25957]: pgmap v220: 65 pgs: 65 active+clean; 577 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1023 B/s wr, 3 op/s 2024-09-20T22:17:15.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:15 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:17:16.738 INFO:teuthology.orchestra.run.smithi037.stdout: 2024-09-20T22:17:16.738 INFO:teuthology.orchestra.run.smithi037.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-20T22:10:35.834131Z", "last_refresh": "2024-09-20T22:14:24.829022Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-20T22:10:29.668492Z", "last_refresh": "2024-09-20T22:14:24.829088Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-09-20T22:10:32.691543Z", "last_refresh": "2024-09-20T22:14:24.829152Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-09-20T22:17:05.197680Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-20T22:17:05.191626Z", "last_refresh": "2024-09-20T22:17:14.484476Z", "running": 1, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-20T22:10:28.369386Z", "last_refresh": "2024-09-20T22:14:24.828948Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi037:172.21.15.37=smithi037", "smithi194:172.21.15.194=smithi194"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-20T22:11:19.388507Z", "last_refresh": "2024-09-20T22:14:24.828805Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-20T22:10:34.364687Z", "last_refresh": "2024-09-20T22:14:24.829215Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-09-20T22:14:24.829341Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-20T22:10:31.440666Z", "last_refresh": "2024-09-20T22:14:24.829278Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-20T22:17:17.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:17 smithi194 ceph-mon[25957]: pgmap v221: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-09-20T22:17:17.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:17 smithi194 ceph-mon[25957]: from='client.14500 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-20T22:17:17.905 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:17 smithi037 ceph-mon[23718]: pgmap v221: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-09-20T22:17:17.905 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:17 smithi037 ceph-mon[23718]: from='client.14500 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-20T22:17:17.936 INFO:tasks.cephadm:mds.foofs has 1/2 2024-09-20T22:17:18.937 DEBUG:teuthology.orchestra.run.smithi037:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -- ceph orch ls -f json 2024-09-20T22:17:19.412 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:19 smithi037 ceph-mon[23718]: pgmap v222: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 4 op/s 2024-09-20T22:17:19.412 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:19 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:17:19.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:19 smithi194 ceph-mon[25957]: pgmap v222: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 4 op/s 2024-09-20T22:17:19.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:19 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:17:21.258 INFO:teuthology.orchestra.run.smithi037.stdout: 2024-09-20T22:17:21.258 INFO:teuthology.orchestra.run.smithi037.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-20T22:10:35.834131Z", "last_refresh": "2024-09-20T22:17:20.832404Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-20T22:10:29.668492Z", "last_refresh": "2024-09-20T22:17:14.483878Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-09-20T22:10:32.691543Z", "last_refresh": "2024-09-20T22:17:20.832618Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-09-20T22:17:05.197680Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-20T22:17:05.191626Z", "last_refresh": "2024-09-20T22:17:14.484476Z", "running": 2, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-20T22:10:28.369386Z", "last_refresh": "2024-09-20T22:17:14.484017Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi037:172.21.15.37=smithi037", "smithi194:172.21.15.194=smithi194"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-20T22:11:19.388507Z", "last_refresh": "2024-09-20T22:17:14.484091Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-20T22:10:34.364687Z", "last_refresh": "2024-09-20T22:17:14.484158Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-09-20T22:17:14.484224Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-20T22:10:31.440666Z", "last_refresh": "2024-09-20T22:17:20.832837Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-20T22:17:21.999 INFO:tasks.cephadm:mds.foofs has 2/2 2024-09-20T22:17:22.000 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-20T22:17:22.008 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi037.front.sepia.ceph.com 2024-09-20T22:17:22.008 DEBUG:teuthology.orchestra.run.smithi037:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -- bash -c 'ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2' 2024-09-20T22:17:22.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:21 smithi194 ceph-mon[25957]: pgmap v223: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-20T22:17:22.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:21 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:17:22.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:21 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:17:22.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:21 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:17:22.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:21 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:17:22.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:21 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:17:22.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:21 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:17:22.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:21 smithi037 ceph-mon[23718]: pgmap v223: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-20T22:17:22.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:21 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:17:22.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:21 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:17:22.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:21 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:17:22.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:21 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:17:22.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:21 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:17:22.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:21 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:17:23.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:22 smithi194 ceph-mon[25957]: from='client.14504 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-20T22:17:23.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:22 smithi037 ceph-mon[23718]: from='client.14504 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-20T22:17:23.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:23 smithi037 ceph-mon[23718]: pgmap v224: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-20T22:17:23.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:23 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:17:24.271 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:23 smithi194 ceph-mon[25957]: pgmap v224: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-20T22:17:24.271 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:23 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:17:25.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:24 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd pool create", "pool": ".nfs"}]: dispatch 2024-09-20T22:17:25.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:24 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd pool create", "pool": ".nfs"}]: dispatch 2024-09-20T22:17:25.892 INFO:teuthology.orchestra.run.smithi037.stdout:NFS Cluster Created Successfully 2024-09-20T22:17:26.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:25 smithi037 ceph-mon[23718]: from='client.14508 -' entity='client.admin' cmd=[{"prefix": "nfs cluster create", "cluster_id": "foo", "placement": "2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:17:26.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:25 smithi037 ceph-mon[23718]: pgmap v225: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 2 op/s 2024-09-20T22:17:26.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:25 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "osd pool create", "pool": ".nfs"}]': finished 2024-09-20T22:17:26.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:25 smithi037 ceph-mon[23718]: osdmap e65: 8 total, 8 up, 8 in 2024-09-20T22:17:26.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:25 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]: dispatch 2024-09-20T22:17:26.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:25 smithi194 ceph-mon[25957]: from='client.14508 -' entity='client.admin' cmd=[{"prefix": "nfs cluster create", "cluster_id": "foo", "placement": "2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:17:26.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:25 smithi194 ceph-mon[25957]: pgmap v225: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 2 op/s 2024-09-20T22:17:26.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:25 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "osd pool create", "pool": ".nfs"}]': finished 2024-09-20T22:17:26.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:25 smithi194 ceph-mon[25957]: osdmap e65: 8 total, 8 up, 8 in 2024-09-20T22:17:26.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:25 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]: dispatch 2024-09-20T22:17:26.631 DEBUG:teuthology.orchestra.run.smithi037:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -- bash -c 'ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake' 2024-09-20T22:17:27.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:26 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]': finished 2024-09-20T22:17:27.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:26 smithi194 ceph-mon[25957]: osdmap e66: 8 total, 8 up, 8 in 2024-09-20T22:17:27.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:26 smithi194 ceph-mon[25957]: Saving service nfs.foo spec with placement count:2 2024-09-20T22:17:27.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:26 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:17:27.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:26 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:17:27.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:26 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:17:27.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:26 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:17:27.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:26 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:17:27.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:26 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:17:27.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:26 smithi194 ceph-mon[25957]: Creating key for client.nfs.foo.0.0.smithi037.spabdc 2024-09-20T22:17:27.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:26 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi037.spabdc", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-20T22:17:27.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:26 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi037.spabdc", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-09-20T22:17:27.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:26 smithi194 ceph-mon[25957]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-09-20T22:17:27.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:26 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-20T22:17:27.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:26 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-20T22:17:27.096 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:26 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:17:27.096 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:26 smithi194 ceph-mon[25957]: pgmap v228: 97 pgs: 1 creating+peering, 31 unknown, 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-20T22:17:27.096 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:26 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-20T22:17:27.096 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:26 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-20T22:17:27.096 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:26 smithi194 ceph-mon[25957]: Rados config object exists: conf-nfs.foo 2024-09-20T22:17:27.096 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:26 smithi194 ceph-mon[25957]: Creating key for client.nfs.foo.0.0.smithi037.spabdc-rgw 2024-09-20T22:17:27.096 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:26 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi037.spabdc-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-20T22:17:27.096 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:26 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi037.spabdc-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-20T22:17:27.096 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:26 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:17:27.096 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:26 smithi194 ceph-mon[25957]: Deploying daemon nfs.foo.0.0.smithi037.spabdc on smithi037 2024-09-20T22:17:27.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:26 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]': finished 2024-09-20T22:17:27.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:26 smithi037 ceph-mon[23718]: osdmap e66: 8 total, 8 up, 8 in 2024-09-20T22:17:27.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:26 smithi037 ceph-mon[23718]: Saving service nfs.foo spec with placement count:2 2024-09-20T22:17:27.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:26 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:17:27.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:26 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:17:27.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:26 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:17:27.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:26 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:17:27.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:26 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:17:27.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:26 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:17:27.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:26 smithi037 ceph-mon[23718]: Creating key for client.nfs.foo.0.0.smithi037.spabdc 2024-09-20T22:17:27.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:26 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi037.spabdc", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-20T22:17:27.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:26 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi037.spabdc", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-09-20T22:17:27.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:26 smithi037 ceph-mon[23718]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-09-20T22:17:27.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:26 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-20T22:17:27.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:26 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-20T22:17:27.164 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:26 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:17:27.164 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:26 smithi037 ceph-mon[23718]: pgmap v228: 97 pgs: 1 creating+peering, 31 unknown, 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-20T22:17:27.164 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:26 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-20T22:17:27.164 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:26 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-20T22:17:27.164 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:26 smithi037 ceph-mon[23718]: Rados config object exists: conf-nfs.foo 2024-09-20T22:17:27.164 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:26 smithi037 ceph-mon[23718]: Creating key for client.nfs.foo.0.0.smithi037.spabdc-rgw 2024-09-20T22:17:27.164 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:26 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi037.spabdc-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-20T22:17:27.164 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:26 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi037.spabdc-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-20T22:17:27.164 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:26 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:17:27.164 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:26 smithi037 ceph-mon[23718]: Deploying daemon nfs.foo.0.0.smithi037.spabdc on smithi037 2024-09-20T22:17:28.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:27 smithi037 ceph-mon[23718]: osdmap e67: 8 total, 8 up, 8 in 2024-09-20T22:17:28.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:27 smithi037 ceph-mon[23718]: mgrmap e20: smithi037.lykblh(active, since 5m), standbys: smithi194.bkcglt 2024-09-20T22:17:28.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:27 smithi194 ceph-mon[25957]: osdmap e67: 8 total, 8 up, 8 in 2024-09-20T22:17:28.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:27 smithi194 ceph-mon[25957]: mgrmap e20: smithi037.lykblh(active, since 5m), standbys: smithi194.bkcglt 2024-09-20T22:17:29.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:28 smithi037 ceph-mon[23718]: pgmap v230: 97 pgs: 1 creating+peering, 8 unknown, 88 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-09-20T22:17:29.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:28 smithi194 ceph-mon[25957]: pgmap v230: 97 pgs: 1 creating+peering, 8 unknown, 88 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-09-20T22:17:29.751 INFO:teuthology.orchestra.run.smithi037.stderr:Invalid command: Unexpected argument '--clusterid' 2024-09-20T22:17:29.751 INFO:teuthology.orchestra.run.smithi037.stderr:nfs export create cephfs [] [--readonly] [--client_addr ...] [--squash ] : Create a CephFS export 2024-09-20T22:17:29.751 INFO:teuthology.orchestra.run.smithi037.stderr:Error EINVAL: invalid command 2024-09-20T22:17:30.107 INFO:teuthology.orchestra.run.smithi037.stdout:{ 2024-09-20T22:17:30.107 INFO:teuthology.orchestra.run.smithi037.stdout: "bind": "/fake", 2024-09-20T22:17:30.107 INFO:teuthology.orchestra.run.smithi037.stdout: "fs": "foofs", 2024-09-20T22:17:30.107 INFO:teuthology.orchestra.run.smithi037.stdout: "path": "/", 2024-09-20T22:17:30.107 INFO:teuthology.orchestra.run.smithi037.stdout: "cluster": "foo", 2024-09-20T22:17:30.107 INFO:teuthology.orchestra.run.smithi037.stdout: "mode": "RW" 2024-09-20T22:17:30.107 INFO:teuthology.orchestra.run.smithi037.stdout:} 2024-09-20T22:17:30.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:30 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2024-09-20T22:17:30.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:30 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2024-09-20T22:17:30.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:30 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:17:30.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:30 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi194.qzjyej", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-20T22:17:30.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:30 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi194.qzjyej", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-09-20T22:17:30.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:30 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-20T22:17:30.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:30 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-20T22:17:30.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:30 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:17:30.880 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:30 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2024-09-20T22:17:30.881 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:30 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2024-09-20T22:17:30.881 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:30 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:17:30.881 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:30 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi194.qzjyej", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-20T22:17:30.881 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:30 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi194.qzjyej", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-09-20T22:17:30.881 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:30 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-20T22:17:30.881 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:30 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-20T22:17:30.881 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:30 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:17:30.907 DEBUG:teuthology.orchestra.run.smithi037:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -- bash -c 'while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done' 2024-09-20T22:17:31.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:31 smithi194 ceph-mon[25957]: from='client.14530 -' entity='client.admin' cmd=[{"prefix": "nfs export create cephfs", "fsname": "foofs", "cluster_id": "foo", "pseudo_path": "/fake", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:17:31.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:31 smithi194 ceph-mon[25957]: Creating key for client.nfs.foo.1.0.smithi194.qzjyej 2024-09-20T22:17:31.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:31 smithi194 ceph-mon[25957]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-09-20T22:17:31.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:31 smithi194 ceph-mon[25957]: pgmap v231: 97 pgs: 1 creating+peering, 96 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-09-20T22:17:31.870 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:31 smithi037 ceph-mon[23718]: from='client.14530 -' entity='client.admin' cmd=[{"prefix": "nfs export create cephfs", "fsname": "foofs", "cluster_id": "foo", "pseudo_path": "/fake", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:17:31.871 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:31 smithi037 ceph-mon[23718]: Creating key for client.nfs.foo.1.0.smithi194.qzjyej 2024-09-20T22:17:31.871 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:31 smithi037 ceph-mon[23718]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-09-20T22:17:31.871 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:31 smithi037 ceph-mon[23718]: pgmap v231: 97 pgs: 1 creating+peering, 96 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-09-20T22:17:33.755 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:33 smithi194 ceph-mon[25957]: pgmap v232: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 139 B/s rd, 279 B/s wr, 0 op/s 2024-09-20T22:17:33.755 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:33 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-20T22:17:33.755 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:33 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-20T22:17:33.755 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:33 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi194.qzjyej-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-20T22:17:33.755 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:33 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi194.qzjyej-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-20T22:17:33.755 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:33 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:17:33.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:33 smithi037 ceph-mon[23718]: pgmap v232: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 139 B/s rd, 279 B/s wr, 0 op/s 2024-09-20T22:17:33.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:33 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-20T22:17:33.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:33 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-20T22:17:33.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:33 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi194.qzjyej-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-20T22:17:33.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:33 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi194.qzjyej-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-20T22:17:33.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:33 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:17:34.764 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:34 smithi194 ceph-mon[25957]: from='client.14546 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:17:34.765 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:34 smithi194 ceph-mon[25957]: Rados config object exists: conf-nfs.foo 2024-09-20T22:17:34.765 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:34 smithi194 ceph-mon[25957]: Creating key for client.nfs.foo.1.0.smithi194.qzjyej-rgw 2024-09-20T22:17:34.765 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:34 smithi194 ceph-mon[25957]: Deploying daemon nfs.foo.1.0.smithi194.qzjyej on smithi194 2024-09-20T22:17:34.814 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:34 smithi037 ceph-mon[23718]: from='client.14546 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:17:34.814 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:34 smithi037 ceph-mon[23718]: Rados config object exists: conf-nfs.foo 2024-09-20T22:17:34.814 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:34 smithi037 ceph-mon[23718]: Creating key for client.nfs.foo.1.0.smithi194.qzjyej-rgw 2024-09-20T22:17:34.814 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:34 smithi037 ceph-mon[23718]: Deploying daemon nfs.foo.1.0.smithi194.qzjyej on smithi194 2024-09-20T22:17:35.662 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:35 smithi037 ceph-mon[23718]: pgmap v233: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1.2 KiB/s wr, 3 op/s 2024-09-20T22:17:35.662 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:35 smithi037 ceph-mon[23718]: from='client.14560 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:17:35.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:35 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:17:35.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:35 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:17:35.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:35 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:17:35.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:35 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:17:35.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:35 smithi194 ceph-mon[25957]: pgmap v233: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1.2 KiB/s wr, 3 op/s 2024-09-20T22:17:35.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:35 smithi194 ceph-mon[25957]: from='client.14560 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:17:35.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:35 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:17:35.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:35 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:17:35.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:35 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:17:35.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:35 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:17:36.807 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:36 smithi194 ceph-mon[25957]: from='client.14576 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:17:36.849 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:36 smithi037 ceph-mon[23718]: from='client.14576 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:17:37.662 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:37 smithi037 ceph-mon[23718]: pgmap v234: 97 pgs: 97 active+clean; 580 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1.5 KiB/s wr, 3 op/s 2024-09-20T22:17:37.797 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:37 smithi194 ceph-mon[25957]: pgmap v234: 97 pgs: 97 active+clean; 580 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1.5 KiB/s wr, 3 op/s 2024-09-20T22:17:38.834 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:38 smithi037 ceph-mon[23718]: from='client.14580 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:17:38.835 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:38 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:17:38.835 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:38 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1f", "id": [7, 2]}]: dispatch 2024-09-20T22:17:38.835 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:38 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2024-09-20T22:17:38.835 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:38 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2024-09-20T22:17:38.835 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:38 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2024-09-20T22:17:38.835 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:38 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi037.lykblh/trash_purge_schedule"}]: dispatch 2024-09-20T22:17:38.835 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:38 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi037.lykblh/mirror_snapshot_schedule"}]: dispatch 2024-09-20T22:17:38.835 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:38 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:17:38.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:38 smithi194 ceph-mon[25957]: from='client.14580 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:17:38.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:38 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:17:38.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:38 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1f", "id": [7, 2]}]: dispatch 2024-09-20T22:17:38.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:38 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2024-09-20T22:17:38.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:38 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2024-09-20T22:17:38.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:38 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2024-09-20T22:17:38.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:38 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi037.lykblh/trash_purge_schedule"}]: dispatch 2024-09-20T22:17:38.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:38 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi037.lykblh/mirror_snapshot_schedule"}]: dispatch 2024-09-20T22:17:38.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:38 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:17:39.805 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:39 smithi037 ceph-mon[23718]: pgmap v235: 97 pgs: 97 active+clean; 580 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.6 KiB/s rd, 2.0 KiB/s wr, 6 op/s 2024-09-20T22:17:39.805 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:39 smithi037 ceph-mon[23718]: from='client.14584 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:17:39.806 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:39 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1f", "id": [7, 2]}]': finished 2024-09-20T22:17:39.806 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:39 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]': finished 2024-09-20T22:17:39.806 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:39 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]': finished 2024-09-20T22:17:39.806 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:39 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]': finished 2024-09-20T22:17:39.806 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:39 smithi037 ceph-mon[23718]: osdmap e68: 8 total, 8 up, 8 in 2024-09-20T22:17:39.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:39 smithi194 ceph-mon[25957]: pgmap v235: 97 pgs: 97 active+clean; 580 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.6 KiB/s rd, 2.0 KiB/s wr, 6 op/s 2024-09-20T22:17:39.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:39 smithi194 ceph-mon[25957]: from='client.14584 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:17:39.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:39 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1f", "id": [7, 2]}]': finished 2024-09-20T22:17:39.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:39 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]': finished 2024-09-20T22:17:39.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:39 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]': finished 2024-09-20T22:17:39.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:39 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]': finished 2024-09-20T22:17:39.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:39 smithi194 ceph-mon[25957]: osdmap e68: 8 total, 8 up, 8 in 2024-09-20T22:17:40.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:40 smithi194 ceph-mon[25957]: from='client.14588 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:17:40.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:40 smithi194 ceph-mon[25957]: osdmap e69: 8 total, 8 up, 8 in 2024-09-20T22:17:40.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:40 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:17:40.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:40 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:17:40.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:40 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:17:40.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:40 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:17:40.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:40 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:17:40.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:40 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:17:40.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:40 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:17:40.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:40 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:17:40.896 INFO:teuthology.orchestra.run.smithi037.stdout:nfs.foo ?:2049 2/2 2s ago 14s count:2 2024-09-20T22:17:40.896 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:40 smithi037 ceph-mon[23718]: from='client.14588 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:17:40.896 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:40 smithi037 ceph-mon[23718]: osdmap e69: 8 total, 8 up, 8 in 2024-09-20T22:17:40.896 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:40 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:17:40.896 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:40 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:17:40.896 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:40 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:17:40.897 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:40 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:17:40.897 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:40 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:17:40.897 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:40 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:17:40.897 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:40 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:17:40.897 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:40 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:17:41.683 INFO:teuthology.run_tasks:Running task vip.exec... 2024-09-20T22:17:41.692 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi037.front.sepia.ceph.com 2024-09-20T22:17:41.692 DEBUG:teuthology.orchestra.run.smithi037:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'mkdir /mnt/foo' 2024-09-20T22:17:41.723 INFO:teuthology.orchestra.run.smithi037.stderr:+ mkdir /mnt/foo 2024-09-20T22:17:41.735 DEBUG:teuthology.orchestra.run.smithi037:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done' 2024-09-20T22:17:41.801 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:41 smithi037 ceph-mon[23718]: pgmap v238: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.1 KiB/s wr, 8 op/s 2024-09-20T22:17:41.804 INFO:teuthology.orchestra.run.smithi037.stderr:++ hostname 2024-09-20T22:17:41.804 INFO:teuthology.orchestra.run.smithi037.stderr:+ mount -t nfs smithi037:/fake /mnt/foo -o sync 2024-09-20T22:17:41.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:41 smithi194 ceph-mon[25957]: pgmap v238: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.1 KiB/s wr, 8 op/s 2024-09-20T22:17:42.253 DEBUG:teuthology.orchestra.run.smithi037:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'echo test > /mnt/foo/testfile' 2024-09-20T22:17:42.319 INFO:teuthology.orchestra.run.smithi037.stderr:+ echo test 2024-09-20T22:17:42.329 DEBUG:teuthology.orchestra.run.smithi037:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c sync 2024-09-20T22:17:42.391 INFO:teuthology.orchestra.run.smithi037.stderr:+ sync 2024-09-20T22:17:42.609 INFO:teuthology.run_tasks:Running task parallel... 2024-09-20T22:17:42.619 INFO:teuthology.task.parallel:starting parallel... 2024-09-20T22:17:42.619 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-09-20T22:17:42.620 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-09-20T22:17:42.620 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi037.front.sepia.ceph.com 2024-09-20T22:17:42.620 DEBUG:teuthology.orchestra.run.smithi037:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-09-20T22:17:42.621 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-09-20T22:17:42.621 INFO:teuthology.task.sequential:In sequential, running task exec... 2024-09-20T22:17:42.622 INFO:teuthology.task.exec:Executing custom commands... 2024-09-20T22:17:42.622 INFO:teuthology.task.exec:Running commands on role host.a host ubuntu@smithi037.front.sepia.ceph.com 2024-09-20T22:17:42.622 DEBUG:teuthology.orchestra.run.smithi037:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'cd /mnt/foo && dbench 5 -t 600 || true' 2024-09-20T22:17:42.688 INFO:teuthology.orchestra.run.smithi037.stdout:dbench version 4.00 - Copyright Andrew Tridgell 1999-2004 2024-09-20T22:17:42.688 INFO:teuthology.orchestra.run.smithi037.stdout: 2024-09-20T22:17:42.688 INFO:teuthology.orchestra.run.smithi037.stdout:Running for 600 seconds with load '/usr/share/dbench/client.txt' and minimum warmup 120 secs 2024-09-20T22:17:42.689 INFO:teuthology.orchestra.run.smithi037.stdout:4 of 5 processes prepared for launch 0 sec 2024-09-20T22:17:42.790 INFO:teuthology.orchestra.run.smithi037.stdout:5 of 5 processes prepared for launch 0 sec 2024-09-20T22:17:42.790 INFO:teuthology.orchestra.run.smithi037.stdout:releasing clients 2024-09-20T22:17:42.803 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:42 smithi037 ceph-mon[23718]: from='client.14592 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:17:42.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:42 smithi194 ceph-mon[25957]: from='client.14592 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:17:43.625 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:43 smithi037 ceph-mon[23718]: pgmap v239: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.5 KiB/s rd, 2.1 KiB/s wr, 6 op/s 2024-09-20T22:17:43.625 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:43 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:17:43.789 INFO:teuthology.orchestra.run.smithi037.stdout: 5 127 28.36 MB/sec warmup 1 sec latency 38.519 ms 2024-09-20T22:17:43.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:43 smithi194 ceph-mon[25957]: pgmap v239: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.5 KiB/s rd, 2.1 KiB/s wr, 6 op/s 2024-09-20T22:17:43.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:43 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:17:44.790 INFO:teuthology.orchestra.run.smithi037.stdout: 5 229 21.92 MB/sec warmup 2 sec latency 65.373 ms 2024-09-20T22:17:45.688 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:45 smithi037 ceph-mon[23718]: pgmap v240: 97 pgs: 97 active+clean; 4.9 MiB data, 52 MiB used, 715 GiB / 715 GiB avail; 3.2 KiB/s rd, 564 KiB/s wr, 14 op/s; 71 B/s, 0 keys/s, 0 objects/s recovering 2024-09-20T22:17:45.790 INFO:teuthology.orchestra.run.smithi037.stdout: 5 322 19.49 MB/sec warmup 3 sec latency 70.890 ms 2024-09-20T22:17:45.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:45 smithi194 ceph-mon[25957]: pgmap v240: 97 pgs: 97 active+clean; 4.9 MiB data, 52 MiB used, 715 GiB / 715 GiB avail; 3.2 KiB/s rd, 564 KiB/s wr, 14 op/s; 71 B/s, 0 keys/s, 0 objects/s recovering 2024-09-20T22:17:46.124 DEBUG:teuthology.orchestra.run.smithi037:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-09-20T22:17:46.790 INFO:teuthology.orchestra.run.smithi037.stdout: 5 447 19.61 MB/sec warmup 4 sec latency 31.535 ms 2024-09-20T22:17:47.790 INFO:teuthology.orchestra.run.smithi037.stdout: 5 571 19.42 MB/sec warmup 5 sec latency 76.313 ms 2024-09-20T22:17:47.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:47 smithi194 ceph-mon[25957]: pgmap v241: 97 pgs: 97 active+clean; 14 MiB data, 55 MiB used, 715 GiB / 715 GiB avail; 0 B/s rd, 1.8 MiB/s wr, 115 op/s; 71 B/s, 0 keys/s, 0 objects/s recovering 2024-09-20T22:17:47.846 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:47 smithi037 ceph-mon[23718]: pgmap v241: 97 pgs: 97 active+clean; 14 MiB data, 55 MiB used, 715 GiB / 715 GiB avail; 0 B/s rd, 1.8 MiB/s wr, 115 op/s; 71 B/s, 0 keys/s, 0 objects/s recovering 2024-09-20T22:17:48.790 INFO:teuthology.orchestra.run.smithi037.stdout: 5 853 17.99 MB/sec warmup 6 sec latency 47.210 ms 2024-09-20T22:17:49.636 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:49 smithi037 ceph-mon[23718]: pgmap v242: 97 pgs: 97 active+clean; 94 MiB data, 122 MiB used, 715 GiB / 715 GiB avail; 223 B/s rd, 9.8 MiB/s wr, 339 op/s; 62 B/s, 0 keys/s, 0 objects/s recovering 2024-09-20T22:17:49.730 DEBUG:teuthology.orchestra.run.smithi037:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph config set global log_to_journald false --force' 2024-09-20T22:17:49.790 INFO:teuthology.orchestra.run.smithi037.stdout: 5 1143 16.43 MB/sec warmup 7 sec latency 78.685 ms 2024-09-20T22:17:49.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:49 smithi194 ceph-mon[25957]: pgmap v242: 97 pgs: 97 active+clean; 94 MiB data, 122 MiB used, 715 GiB / 715 GiB avail; 223 B/s rd, 9.8 MiB/s wr, 339 op/s; 62 B/s, 0 keys/s, 0 objects/s recovering 2024-09-20T22:17:50.790 INFO:teuthology.orchestra.run.smithi037.stdout: 5 1978 16.56 MB/sec warmup 8 sec latency 32.619 ms 2024-09-20T22:17:51.659 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:51 smithi037 ceph-mon[23718]: pgmap v243: 97 pgs: 97 active+clean; 103 MiB data, 157 MiB used, 715 GiB / 715 GiB avail; 100 B/s rd, 9.8 MiB/s wr, 403 op/s; 56 B/s, 0 keys/s, 0 objects/s recovering 2024-09-20T22:17:51.790 INFO:teuthology.orchestra.run.smithi037.stdout: 5 2605 16.35 MB/sec warmup 9 sec latency 70.870 ms 2024-09-20T22:17:51.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:51 smithi194 ceph-mon[25957]: pgmap v243: 97 pgs: 97 active+clean; 103 MiB data, 157 MiB used, 715 GiB / 715 GiB avail; 100 B/s rd, 9.8 MiB/s wr, 403 op/s; 56 B/s, 0 keys/s, 0 objects/s recovering 2024-09-20T22:17:52.790 INFO:teuthology.orchestra.run.smithi037.stdout: 5 3147 17.09 MB/sec warmup 10 sec latency 63.920 ms 2024-09-20T22:17:53.789 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:53 smithi037 ceph-mon[23718]: pgmap v244: 97 pgs: 97 active+clean; 106 MiB data, 172 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 8.7 MiB/s wr, 356 op/s; 47 B/s, 0 keys/s, 0 objects/s recovering 2024-09-20T22:17:53.790 INFO:teuthology.orchestra.run.smithi037.stdout: 5 3671 17.64 MB/sec warmup 11 sec latency 38.698 ms 2024-09-20T22:17:53.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:53 smithi194 ceph-mon[25957]: pgmap v244: 97 pgs: 97 active+clean; 106 MiB data, 172 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 8.7 MiB/s wr, 356 op/s; 47 B/s, 0 keys/s, 0 objects/s recovering 2024-09-20T22:17:54.325 DEBUG:teuthology.orchestra.run.smithi037:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph mgr module enable nfs --force' 2024-09-20T22:17:54.790 INFO:teuthology.orchestra.run.smithi037.stdout: 5 3877 16.59 MB/sec warmup 12 sec latency 62.178 ms 2024-09-20T22:17:55.597 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:55 smithi037 ceph-mon[23718]: pgmap v245: 97 pgs: 97 active+clean; 124 MiB data, 205 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 12 MiB/s wr, 624 op/s; 47 B/s, 0 keys/s, 0 objects/s recovering 2024-09-20T22:17:55.790 INFO:teuthology.orchestra.run.smithi037.stdout: 5 4047 15.43 MB/sec warmup 13 sec latency 115.922 ms 2024-09-20T22:17:55.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:55 smithi194 ceph-mon[25957]: pgmap v245: 97 pgs: 97 active+clean; 124 MiB data, 205 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 12 MiB/s wr, 624 op/s; 47 B/s, 0 keys/s, 0 objects/s recovering 2024-09-20T22:17:56.790 INFO:teuthology.orchestra.run.smithi037.stdout: 5 4192 14.36 MB/sec warmup 14 sec latency 280.682 ms 2024-09-20T22:17:57.600 INFO:teuthology.orchestra.run.smithi037.stderr:module 'nfs' is already enabled 2024-09-20T22:17:57.758 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:57 smithi037 ceph-mon[23718]: pgmap v246: 97 pgs: 97 active+clean; 125 MiB data, 213 MiB used, 715 GiB / 715 GiB avail; 3.3 KiB/s rd, 12 MiB/s wr, 635 op/s 2024-09-20T22:17:57.758 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:57 smithi037 ceph-mon[23718]: from='client.? 172.21.15.37:0/841089444' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-09-20T22:17:57.790 INFO:teuthology.orchestra.run.smithi037.stdout: 5 4399 13.54 MB/sec warmup 15 sec latency 217.864 ms 2024-09-20T22:17:58.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:57 smithi194 ceph-mon[25957]: pgmap v246: 97 pgs: 97 active+clean; 125 MiB data, 213 MiB used, 715 GiB / 715 GiB avail; 3.3 KiB/s rd, 12 MiB/s wr, 635 op/s 2024-09-20T22:17:58.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:57 smithi194 ceph-mon[25957]: from='client.? 172.21.15.37:0/841089444' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-09-20T22:17:58.778 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:58 smithi037 ceph-mon[23718]: from='client.? 172.21.15.37:0/841089444' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "nfs", "force": true}]': finished 2024-09-20T22:17:58.778 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:58 smithi037 ceph-mon[23718]: mgrmap e21: smithi037.lykblh(active, since 6m), standbys: smithi194.bkcglt 2024-09-20T22:17:58.790 INFO:teuthology.orchestra.run.smithi037.stdout: 5 4754 13.17 MB/sec warmup 16 sec latency 83.562 ms 2024-09-20T22:17:59.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:58 smithi194 ceph-mon[25957]: from='client.? 172.21.15.37:0/841089444' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "nfs", "force": true}]': finished 2024-09-20T22:17:59.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:58 smithi194 ceph-mon[25957]: mgrmap e21: smithi037.lykblh(active, since 6m), standbys: smithi194.bkcglt 2024-09-20T22:17:59.650 DEBUG:teuthology.orchestra.run.smithi037:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-09-20T22:17:59.790 INFO:teuthology.orchestra.run.smithi037.stdout: 5 5211 13.33 MB/sec warmup 17 sec latency 71.903 ms 2024-09-20T22:17:59.795 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:17:59 smithi037 ceph-mon[23718]: pgmap v247: 97 pgs: 97 active+clean; 130 MiB data, 246 MiB used, 715 GiB / 715 GiB avail; 3.5 KiB/s rd, 12 MiB/s wr, 664 op/s 2024-09-20T22:17:59.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:17:59 smithi194 ceph-mon[25957]: pgmap v247: 97 pgs: 97 active+clean; 130 MiB data, 246 MiB used, 715 GiB / 715 GiB avail; 3.5 KiB/s rd, 12 MiB/s wr, 664 op/s 2024-09-20T22:18:00.790 INFO:teuthology.orchestra.run.smithi037.stdout: 5 5748 12.78 MB/sec warmup 18 sec latency 120.584 ms 2024-09-20T22:18:01.790 INFO:teuthology.orchestra.run.smithi037.stdout: 5 5991 12.43 MB/sec warmup 19 sec latency 84.222 ms 2024-09-20T22:18:01.841 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:18:01 smithi037 ceph-mon[23718]: pgmap v248: 97 pgs: 97 active+clean; 134 MiB data, 262 MiB used, 715 GiB / 715 GiB avail; 3.3 KiB/s rd, 6.3 MiB/s wr, 562 op/s 2024-09-20T22:18:02.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:18:01 smithi194 ceph-mon[25957]: pgmap v248: 97 pgs: 97 active+clean; 134 MiB data, 262 MiB used, 715 GiB / 715 GiB avail; 3.3 KiB/s rd, 6.3 MiB/s wr, 562 op/s 2024-09-20T22:18:02.605 INFO:teuthology.orchestra.run.smithi037.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-20T22:18:02.790 INFO:teuthology.orchestra.run.smithi037.stdout: 5 6289 12.49 MB/sec warmup 20 sec latency 63.668 ms 2024-09-20T22:18:03.512 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-09-20T22:18:03.512 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi037.front.sepia.ceph.com 2024-09-20T22:18:03.513 DEBUG:teuthology.orchestra.run.smithi037:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done' 2024-09-20T22:18:03.623 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:18:03 smithi037 ceph-mon[23718]: pgmap v249: 97 pgs: 97 active+clean; 133 MiB data, 259 MiB used, 715 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.5 MiB/s wr, 491 op/s 2024-09-20T22:18:03.623 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:18:03 smithi037 ceph-mon[23718]: from='client.14612 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:18:03.623 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:18:03 smithi037 ceph-mon[23718]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-20T22:18:03.623 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:18:03 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:18:03.623 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:18:03 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:18:03.623 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:18:03 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:18:03.623 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:18:03 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:18:03.623 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:18:03 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:18:03.623 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:18:03 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:18:03.790 INFO:teuthology.orchestra.run.smithi037.stdout: 5 6763 12.80 MB/sec warmup 21 sec latency 114.176 ms 2024-09-20T22:18:04.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:18:03 smithi194 ceph-mon[25957]: pgmap v249: 97 pgs: 97 active+clean; 133 MiB data, 259 MiB used, 715 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.5 MiB/s wr, 491 op/s 2024-09-20T22:18:04.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:18:03 smithi194 ceph-mon[25957]: from='client.14612 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:18:04.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:18:03 smithi194 ceph-mon[25957]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-20T22:18:04.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:18:03 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:18:04.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:18:03 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:18:04.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:18:03 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:18:04.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:18:03 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:18:04.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:18:03 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:18:04.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:18:03 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:18:04.630 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:18:04 smithi037 ceph-mon[23718]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-20T22:18:04.790 INFO:teuthology.orchestra.run.smithi037.stdout: 5 7226 13.19 MB/sec warmup 22 sec latency 34.104 ms 2024-09-20T22:18:05.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:18:04 smithi194 ceph-mon[25957]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-20T22:18:05.657 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:18:05 smithi037 ceph-mon[23718]: pgmap v250: 97 pgs: 97 active+clean; 131 MiB data, 299 MiB used, 715 GiB / 715 GiB avail; 6.8 KiB/s rd, 6.9 MiB/s wr, 651 op/s 2024-09-20T22:18:05.790 INFO:teuthology.orchestra.run.smithi037.stdout: 5 7430 12.84 MB/sec warmup 23 sec latency 48.220 ms 2024-09-20T22:18:06.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:18:05 smithi194 ceph-mon[25957]: pgmap v250: 97 pgs: 97 active+clean; 131 MiB data, 299 MiB used, 715 GiB / 715 GiB avail; 6.8 KiB/s rd, 6.9 MiB/s wr, 651 op/s 2024-09-20T22:18:06.791 INFO:teuthology.orchestra.run.smithi037.stdout: 5 7684 12.35 MB/sec warmup 24 sec latency 70.525 ms 2024-09-20T22:18:06.998 INFO:teuthology.orchestra.run.smithi037.stdout:true 2024-09-20T22:18:07.689 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:18:07 smithi037 ceph-mon[23718]: pgmap v251: 97 pgs: 97 active+clean; 131 MiB data, 310 MiB used, 715 GiB / 715 GiB avail; 8.8 KiB/s rd, 3.8 MiB/s wr, 397 op/s 2024-09-20T22:18:07.790 INFO:teuthology.orchestra.run.smithi037.stdout: 5 8034 11.99 MB/sec warmup 25 sec latency 55.354 ms 2024-09-20T22:18:08.085 INFO:teuthology.orchestra.run.smithi037.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T22:18:08.086 INFO:teuthology.orchestra.run.smithi037.stdout:alertmanager.smithi037 smithi037 *:9093,9094 running (5m) 27s ago 6m 24.3M - ba2b418f427c bb407b78f0c9 2024-09-20T22:18:08.086 INFO:teuthology.orchestra.run.smithi037.stdout:crash.smithi037 smithi037 running (6m) 27s ago 6m 6400k - 17.2.0 e1d6a67b021e 88301022bdc8 2024-09-20T22:18:08.086 INFO:teuthology.orchestra.run.smithi037.stdout:crash.smithi194 smithi194 running (6m) 30s ago 6m 6405k - 17.2.0 e1d6a67b021e b553601d7ca5 2024-09-20T22:18:08.086 INFO:teuthology.orchestra.run.smithi037.stdout:grafana.smithi037 smithi037 *:3000 running (5m) 27s ago 6m 52.0M - 8.3.5 dad864ee21e9 3ccb7bb6f7ae 2024-09-20T22:18:08.086 INFO:teuthology.orchestra.run.smithi037.stdout:mds.foofs.smithi037.sdtedi smithi037 running (59s) 27s ago 57s 13.2M - 17.2.0 e1d6a67b021e 0a7898d25b8d 2024-09-20T22:18:08.086 INFO:teuthology.orchestra.run.smithi037.stdout:mds.foofs.smithi194.lkroso smithi194 running (56s) 30s ago 55s 13.4M - 17.2.0 e1d6a67b021e 17736de30008 2024-09-20T22:18:08.086 INFO:teuthology.orchestra.run.smithi037.stdout:mgr.smithi037.lykblh smithi037 *:9283 running (8m) 27s ago 8m 460M - 17.2.0 e1d6a67b021e 3ef1f99a6ba9 2024-09-20T22:18:08.086 INFO:teuthology.orchestra.run.smithi037.stdout:mgr.smithi194.bkcglt smithi194 *:8443,9283 running (6m) 30s ago 6m 413M - 17.2.0 e1d6a67b021e 7cc5497632b6 2024-09-20T22:18:08.086 INFO:teuthology.orchestra.run.smithi037.stdout:mon.smithi037 smithi037 running (8m) 27s ago 8m 43.5M 2048M 17.2.0 e1d6a67b021e 71f591122786 2024-09-20T22:18:08.086 INFO:teuthology.orchestra.run.smithi037.stdout:mon.smithi194 smithi194 running (6m) 30s ago 6m 37.9M 2048M 17.2.0 e1d6a67b021e 0b8814ae6c66 2024-09-20T22:18:08.086 INFO:teuthology.orchestra.run.smithi037.stdout:nfs.foo.0.0.smithi037.spabdc smithi037 *:2049 running (38s) 27s ago 38s 51.1M - 4.0 e1d6a67b021e 858ff9d558a2 2024-09-20T22:18:08.086 INFO:teuthology.orchestra.run.smithi037.stdout:nfs.foo.1.0.smithi194.qzjyej smithi194 *:2049 running (33s) 30s ago 32s 52.8M - 4.0 e1d6a67b021e c07fe0eb7c75 2024-09-20T22:18:08.086 INFO:teuthology.orchestra.run.smithi037.stdout:node-exporter.smithi037 smithi037 *:9100 running (6m) 27s ago 6m 16.5M - 1dbe0e931976 1f92060f034e 2024-09-20T22:18:08.087 INFO:teuthology.orchestra.run.smithi037.stdout:node-exporter.smithi194 smithi194 *:9100 running (6m) 30s ago 6m 17.7M - 1dbe0e931976 3a80a2ef6678 2024-09-20T22:18:08.087 INFO:teuthology.orchestra.run.smithi037.stdout:osd.0 smithi037 running (5m) 27s ago 5m 43.2M 3217M 17.2.0 e1d6a67b021e e54e84fd5030 2024-09-20T22:18:08.087 INFO:teuthology.orchestra.run.smithi037.stdout:osd.1 smithi037 running (4m) 27s ago 4m 42.7M 3217M 17.2.0 e1d6a67b021e 09647c133032 2024-09-20T22:18:08.087 INFO:teuthology.orchestra.run.smithi037.stdout:osd.2 smithi037 running (4m) 27s ago 4m 42.2M 3217M 17.2.0 e1d6a67b021e 6c15b1175256 2024-09-20T22:18:08.087 INFO:teuthology.orchestra.run.smithi037.stdout:osd.3 smithi037 running (3m) 27s ago 3m 39.6M 3217M 17.2.0 e1d6a67b021e 3a22952c6475 2024-09-20T22:18:08.087 INFO:teuthology.orchestra.run.smithi037.stdout:osd.4 smithi194 running (3m) 30s ago 3m 39.2M 3984M 17.2.0 e1d6a67b021e fa107be8915c 2024-09-20T22:18:08.087 INFO:teuthology.orchestra.run.smithi037.stdout:osd.5 smithi194 running (3m) 30s ago 3m 39.1M 3984M 17.2.0 e1d6a67b021e 290b21cbd131 2024-09-20T22:18:08.087 INFO:teuthology.orchestra.run.smithi037.stdout:osd.6 smithi194 running (2m) 30s ago 2m 41.7M 3984M 17.2.0 e1d6a67b021e 27f0720639ed 2024-09-20T22:18:08.087 INFO:teuthology.orchestra.run.smithi037.stdout:osd.7 smithi194 running (2m) 30s ago 2m 41.0M 3984M 17.2.0 e1d6a67b021e 62d163f4fd92 2024-09-20T22:18:08.087 INFO:teuthology.orchestra.run.smithi037.stdout:prometheus.smithi037 smithi037 *:9095 running (5m) 27s ago 5m 61.8M - 514e6a882f6e 32020a1cc2f4 2024-09-20T22:18:08.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:18:07 smithi194 ceph-mon[25957]: pgmap v251: 97 pgs: 97 active+clean; 131 MiB data, 310 MiB used, 715 GiB / 715 GiB avail; 8.8 KiB/s rd, 3.8 MiB/s wr, 397 op/s 2024-09-20T22:18:08.702 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:18:08 smithi037 ceph-mon[23718]: from='client.14616 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:18:08.702 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:18:08 smithi037 ceph-mon[23718]: from='client.24407 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:18:08.789 INFO:teuthology.orchestra.run.smithi037.stdout:{ 2024-09-20T22:18:08.789 INFO:teuthology.orchestra.run.smithi037.stdout: "mon": { 2024-09-20T22:18:08.789 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T22:18:08.789 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:18:08.789 INFO:teuthology.orchestra.run.smithi037.stdout: "mgr": { 2024-09-20T22:18:08.790 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T22:18:08.790 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:18:08.790 INFO:teuthology.orchestra.run.smithi037.stdout: "osd": { 2024-09-20T22:18:08.790 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-20T22:18:08.790 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:18:08.790 INFO:teuthology.orchestra.run.smithi037.stdout: "mds": { 2024-09-20T22:18:08.790 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T22:18:08.790 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:18:08.790 INFO:teuthology.orchestra.run.smithi037.stdout: "overall": { 2024-09-20T22:18:08.790 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-09-20T22:18:08.790 INFO:teuthology.orchestra.run.smithi037.stdout: } 2024-09-20T22:18:08.790 INFO:teuthology.orchestra.run.smithi037.stdout:} 2024-09-20T22:18:08.791 INFO:teuthology.orchestra.run.smithi037.stdout: 5 8381 11.91 MB/sec warmup 26 sec latency 50.358 ms 2024-09-20T22:18:09.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:18:08 smithi194 ceph-mon[25957]: from='client.14616 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:18:09.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:18:08 smithi194 ceph-mon[25957]: from='client.24407 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:18:09.317 INFO:teuthology.orchestra.run.smithi037.stdout:{ 2024-09-20T22:18:09.318 INFO:teuthology.orchestra.run.smithi037.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736", 2024-09-20T22:18:09.318 INFO:teuthology.orchestra.run.smithi037.stdout: "in_progress": true, 2024-09-20T22:18:09.318 INFO:teuthology.orchestra.run.smithi037.stdout: "services_complete": [], 2024-09-20T22:18:09.318 INFO:teuthology.orchestra.run.smithi037.stdout: "progress": "", 2024-09-20T22:18:09.318 INFO:teuthology.orchestra.run.smithi037.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 image" 2024-09-20T22:18:09.318 INFO:teuthology.orchestra.run.smithi037.stdout:} 2024-09-20T22:18:09.729 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:18:09 smithi037 ceph-mon[23718]: from='client.14624 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:18:09.729 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:18:09 smithi037 ceph-mon[23718]: pgmap v252: 97 pgs: 97 active+clean; 148 MiB data, 365 MiB used, 715 GiB / 715 GiB avail; 6.2 KiB/s rd, 6.0 MiB/s wr, 575 op/s 2024-09-20T22:18:09.729 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:18:09 smithi037 ceph-mon[23718]: from='client.? 172.21.15.37:0/2749710497' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:18:09.791 INFO:teuthology.orchestra.run.smithi037.stdout: 5 8808 11.94 MB/sec warmup 27 sec latency 57.895 ms 2024-09-20T22:18:09.827 INFO:teuthology.orchestra.run.smithi037.stdout:HEALTH_OK 2024-09-20T22:18:09.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:18:09 smithi194 ceph-mon[25957]: from='client.14624 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:18:09.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:18:09 smithi194 ceph-mon[25957]: pgmap v252: 97 pgs: 97 active+clean; 148 MiB data, 365 MiB used, 715 GiB / 715 GiB avail; 6.2 KiB/s rd, 6.0 MiB/s wr, 575 op/s 2024-09-20T22:18:09.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:18:09 smithi194 ceph-mon[25957]: from='client.? 172.21.15.37:0/2749710497' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:18:10.760 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:18:10 smithi037 ceph-mon[23718]: from='client.24415 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:18:10.760 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:18:10 smithi037 ceph-mon[23718]: from='client.? 172.21.15.37:0/2859791233' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T22:18:10.791 INFO:teuthology.orchestra.run.smithi037.stdout: 5 9186 11.58 MB/sec warmup 28 sec latency 82.566 ms 2024-09-20T22:18:11.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:18:10 smithi194 ceph-mon[25957]: from='client.24415 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:18:11.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:18:10 smithi194 ceph-mon[25957]: from='client.? 172.21.15.37:0/2859791233' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T22:18:11.690 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:18:11 smithi037 ceph-mon[23718]: pgmap v253: 97 pgs: 97 active+clean; 147 MiB data, 391 MiB used, 715 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.8 MiB/s wr, 485 op/s 2024-09-20T22:18:11.791 INFO:teuthology.orchestra.run.smithi037.stdout: 5 9458 11.40 MB/sec warmup 29 sec latency 152.081 ms 2024-09-20T22:18:12.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:18:11 smithi194 ceph-mon[25957]: pgmap v253: 97 pgs: 97 active+clean; 147 MiB data, 391 MiB used, 715 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.8 MiB/s wr, 485 op/s 2024-09-20T22:18:12.791 INFO:teuthology.orchestra.run.smithi037.stdout: 5 9594 11.16 MB/sec warmup 30 sec latency 56.985 ms 2024-09-20T22:18:13.732 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:18:13 smithi037 ceph-mon[23718]: pgmap v254: 97 pgs: 97 active+clean; 143 MiB data, 396 MiB used, 715 GiB / 715 GiB avail; 10 KiB/s rd, 4.4 MiB/s wr, 417 op/s 2024-09-20T22:18:13.791 INFO:teuthology.orchestra.run.smithi037.stdout: 5 10146 11.60 MB/sec warmup 31 sec latency 45.298 ms 2024-09-20T22:18:14.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:18:13 smithi194 ceph-mon[25957]: pgmap v254: 97 pgs: 97 active+clean; 143 MiB data, 396 MiB used, 715 GiB / 715 GiB avail; 10 KiB/s rd, 4.4 MiB/s wr, 417 op/s 2024-09-20T22:18:14.791 INFO:teuthology.orchestra.run.smithi037.stdout: 5 10739 12.01 MB/sec warmup 32 sec latency 62.288 ms 2024-09-20T22:18:15.762 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:18:15 smithi037 ceph-mon[23718]: pgmap v255: 97 pgs: 97 active+clean; 142 MiB data, 427 MiB used, 715 GiB / 715 GiB avail; 6.5 KiB/s rd, 5.9 MiB/s wr, 593 op/s 2024-09-20T22:18:15.791 INFO:teuthology.orchestra.run.smithi037.stdout: 5 11005 11.85 MB/sec warmup 33 sec latency 42.235 ms 2024-09-20T22:18:16.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:18:15 smithi194 ceph-mon[25957]: pgmap v255: 97 pgs: 97 active+clean; 142 MiB data, 427 MiB used, 715 GiB / 715 GiB avail; 6.5 KiB/s rd, 5.9 MiB/s wr, 593 op/s 2024-09-20T22:18:16.791 INFO:teuthology.orchestra.run.smithi037.stdout: 5 11411 11.57 MB/sec warmup 34 sec latency 33.290 ms 2024-09-20T22:18:17.779 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:18:17 smithi037 ceph-mon[23718]: pgmap v256: 97 pgs: 97 active+clean; 144 MiB data, 411 MiB used, 715 GiB / 715 GiB avail; 8.3 KiB/s rd, 4.5 MiB/s wr, 432 op/s 2024-09-20T22:18:17.791 INFO:teuthology.orchestra.run.smithi037.stdout: 5 11724 11.44 MB/sec warmup 35 sec latency 30.187 ms 2024-09-20T22:18:18.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:18:17 smithi194 ceph-mon[25957]: pgmap v256: 97 pgs: 97 active+clean; 144 MiB data, 411 MiB used, 715 GiB / 715 GiB avail; 8.3 KiB/s rd, 4.5 MiB/s wr, 432 op/s 2024-09-20T22:18:18.791 INFO:teuthology.orchestra.run.smithi037.stdout: 5 12072 11.50 MB/sec warmup 36 sec latency 56.943 ms 2024-09-20T22:18:19.791 INFO:teuthology.orchestra.run.smithi037.stdout: 5 12494 11.31 MB/sec warmup 37 sec latency 56.412 ms 2024-09-20T22:18:19.821 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:18:19 smithi037 ceph-mon[23718]: pgmap v257: 97 pgs: 97 active+clean; 160 MiB data, 427 MiB used, 715 GiB / 715 GiB avail; 6.5 KiB/s rd, 7.0 MiB/s wr, 616 op/s 2024-09-20T22:18:20.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:18:19 smithi194 ceph-mon[25957]: pgmap v257: 97 pgs: 97 active+clean; 160 MiB data, 427 MiB used, 715 GiB / 715 GiB avail; 6.5 KiB/s rd, 7.0 MiB/s wr, 616 op/s 2024-09-20T22:18:20.791 INFO:teuthology.orchestra.run.smithi037.stdout: 5 12934 11.12 MB/sec warmup 38 sec latency 52.333 ms 2024-09-20T22:18:21.791 INFO:teuthology.orchestra.run.smithi037.stdout: 5 13156 11.04 MB/sec warmup 39 sec latency 42.209 ms 2024-09-20T22:18:21.917 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:18:21 smithi037 ceph-mon[23718]: pgmap v258: 97 pgs: 97 active+clean; 159 MiB data, 463 MiB used, 715 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.7 MiB/s wr, 433 op/s 2024-09-20T22:18:22.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:18:21 smithi194 ceph-mon[25957]: pgmap v258: 97 pgs: 97 active+clean; 159 MiB data, 463 MiB used, 715 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.7 MiB/s wr, 433 op/s 2024-09-20T22:18:22.791 INFO:teuthology.orchestra.run.smithi037.stdout: 5 13625 11.36 MB/sec warmup 40 sec latency 93.738 ms 2024-09-20T22:18:23.730 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:18:23 smithi037 ceph-mon[23718]: pgmap v259: 97 pgs: 97 active+clean; 158 MiB data, 468 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 4.8 MiB/s wr, 461 op/s 2024-09-20T22:18:23.791 INFO:teuthology.orchestra.run.smithi037.stdout: 5 14055 11.39 MB/sec warmup 41 sec latency 69.746 ms 2024-09-20T22:18:24.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:18:23 smithi194 ceph-mon[25957]: pgmap v259: 97 pgs: 97 active+clean; 158 MiB data, 468 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 4.8 MiB/s wr, 461 op/s 2024-09-20T22:18:24.791 INFO:teuthology.orchestra.run.smithi037.stdout: 5 14459 11.56 MB/sec warmup 42 sec latency 32.284 ms 2024-09-20T22:18:25.761 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:18:25 smithi037 ceph-mon[23718]: pgmap v260: 97 pgs: 97 active+clean; 160 MiB data, 529 MiB used, 715 GiB / 715 GiB avail; 6.5 KiB/s rd, 6.5 MiB/s wr, 647 op/s 2024-09-20T22:18:25.791 INFO:teuthology.orchestra.run.smithi037.stdout: 5 14894 11.36 MB/sec warmup 43 sec latency 31.131 ms 2024-09-20T22:18:26.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:18:25 smithi194 ceph-mon[25957]: pgmap v260: 97 pgs: 97 active+clean; 160 MiB data, 529 MiB used, 715 GiB / 715 GiB avail; 6.5 KiB/s rd, 6.5 MiB/s wr, 647 op/s 2024-09-20T22:18:26.791 INFO:teuthology.orchestra.run.smithi037.stdout: 5 15174 11.19 MB/sec warmup 44 sec latency 53.788 ms 2024-09-20T22:18:27.791 INFO:teuthology.orchestra.run.smithi037.stdout: 5 15431 11.12 MB/sec warmup 45 sec latency 40.250 ms 2024-09-20T22:18:27.846 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:18:27 smithi037 ceph-mon[23718]: pgmap v261: 97 pgs: 97 active+clean; 162 MiB data, 534 MiB used, 715 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.1 MiB/s wr, 474 op/s 2024-09-20T22:18:28.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:18:27 smithi194 ceph-mon[25957]: pgmap v261: 97 pgs: 97 active+clean; 162 MiB data, 534 MiB used, 715 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.1 MiB/s wr, 474 op/s 2024-09-20T22:18:28.791 INFO:teuthology.orchestra.run.smithi037.stdout: 5 15773 11.15 MB/sec warmup 46 sec latency 62.813 ms 2024-09-20T22:18:29.791 INFO:teuthology.orchestra.run.smithi037.stdout: 5 16157 10.95 MB/sec warmup 47 sec latency 62.940 ms 2024-09-20T22:18:29.873 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:18:29 smithi037 ceph-mon[23718]: pgmap v262: 97 pgs: 97 active+clean; 165 MiB data, 561 MiB used, 715 GiB / 715 GiB avail; 6.9 KiB/s rd, 7.3 MiB/s wr, 669 op/s 2024-09-20T22:18:30.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:18:29 smithi194 ceph-mon[25957]: pgmap v262: 97 pgs: 97 active+clean; 165 MiB data, 561 MiB used, 715 GiB / 715 GiB avail; 6.9 KiB/s rd, 7.3 MiB/s wr, 669 op/s 2024-09-20T22:18:30.793 INFO:teuthology.orchestra.run.smithi037.stdout: 5 16611 10.89 MB/sec warmup 48 sec latency 40.300 ms 2024-09-20T22:18:31.792 INFO:teuthology.orchestra.run.smithi037.stdout: 5 17062 11.12 MB/sec warmup 49 sec latency 77.058 ms 2024-09-20T22:18:31.905 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:18:31 smithi037 ceph-mon[23718]: pgmap v263: 97 pgs: 97 active+clean; 162 MiB data, 582 MiB used, 715 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.7 MiB/s wr, 480 op/s 2024-09-20T22:18:32.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:18:31 smithi194 ceph-mon[25957]: pgmap v263: 97 pgs: 97 active+clean; 162 MiB data, 582 MiB used, 715 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.7 MiB/s wr, 480 op/s 2024-09-20T22:18:32.792 INFO:teuthology.orchestra.run.smithi037.stdout: 5 17502 11.17 MB/sec warmup 50 sec latency 74.677 ms 2024-09-20T22:18:33.792 INFO:teuthology.orchestra.run.smithi037.stdout: 5 17898 11.31 MB/sec warmup 51 sec latency 63.566 ms 2024-09-20T22:18:33.984 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:18:33 smithi037 ceph-mon[23718]: pgmap v264: 97 pgs: 97 active+clean; 161 MiB data, 591 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 4.7 MiB/s wr, 482 op/s 2024-09-20T22:18:33.984 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:18:33 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:18:33.984 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:18:33 smithi037 ceph-mon[23718]: Upgrade: Target is version 19.1.1-367-g9e2ac70b (unknown) 2024-09-20T22:18:33.984 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:18:33 smithi037 ceph-mon[23718]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, digests ['quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5'] 2024-09-20T22:18:33.984 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:18:33 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:18:33.984 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:18:33 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:18:33.984 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:18:33 smithi037 ceph-mon[23718]: Upgrade: Need to upgrade myself (mgr.smithi037.lykblh) 2024-09-20T22:18:34.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:18:33 smithi194 ceph-mon[25957]: pgmap v264: 97 pgs: 97 active+clean; 161 MiB data, 591 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 4.7 MiB/s wr, 482 op/s 2024-09-20T22:18:34.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:18:33 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:18:34.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:18:33 smithi194 ceph-mon[25957]: Upgrade: Target is version 19.1.1-367-g9e2ac70b (unknown) 2024-09-20T22:18:34.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:18:33 smithi194 ceph-mon[25957]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, digests ['quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5'] 2024-09-20T22:18:34.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:18:33 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:18:34.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:18:33 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:18:34.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:18:33 smithi194 ceph-mon[25957]: Upgrade: Need to upgrade myself (mgr.smithi037.lykblh) 2024-09-20T22:18:34.769 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:18:34 smithi037 ceph-mon[23718]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5 on smithi194 2024-09-20T22:18:34.791 INFO:teuthology.orchestra.run.smithi037.stdout: 5 18079 11.19 MB/sec warmup 52 sec latency 62.544 ms 2024-09-20T22:18:35.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:18:34 smithi194 ceph-mon[25957]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5 on smithi194 2024-09-20T22:18:35.789 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:18:35 smithi037 ceph-mon[23718]: pgmap v265: 97 pgs: 97 active+clean; 172 MiB data, 612 MiB used, 715 GiB / 715 GiB avail; 6.5 KiB/s rd, 6.3 MiB/s wr, 558 op/s 2024-09-20T22:18:35.794 INFO:teuthology.orchestra.run.smithi037.stdout: 5 18383 11.01 MB/sec warmup 53 sec latency 71.484 ms 2024-09-20T22:18:36.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:18:35 smithi194 ceph-mon[25957]: pgmap v265: 97 pgs: 97 active+clean; 172 MiB data, 612 MiB used, 715 GiB / 715 GiB avail; 6.5 KiB/s rd, 6.3 MiB/s wr, 558 op/s 2024-09-20T22:18:36.792 INFO:teuthology.orchestra.run.smithi037.stdout: 5 18822 10.96 MB/sec warmup 54 sec latency 71.696 ms 2024-09-20T22:18:37.792 INFO:teuthology.orchestra.run.smithi037.stdout: 5 19448 11.07 MB/sec warmup 55 sec latency 32.994 ms 2024-09-20T22:18:37.798 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:18:37 smithi037 ceph-mon[23718]: pgmap v266: 97 pgs: 97 active+clean; 177 MiB data, 602 MiB used, 715 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.1 MiB/s wr, 469 op/s 2024-09-20T22:18:38.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:18:37 smithi194 ceph-mon[25957]: pgmap v266: 97 pgs: 97 active+clean; 177 MiB data, 602 MiB used, 715 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.1 MiB/s wr, 469 op/s 2024-09-20T22:18:38.792 INFO:teuthology.orchestra.run.smithi037.stdout: 5 20059 11.00 MB/sec warmup 56 sec latency 53.417 ms 2024-09-20T22:18:38.834 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:18:38 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi037.lykblh/trash_purge_schedule"}]: dispatch 2024-09-20T22:18:38.834 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:18:38 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi037.lykblh/mirror_snapshot_schedule"}]: dispatch 2024-09-20T22:18:39.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:18:38 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi037.lykblh/trash_purge_schedule"}]: dispatch 2024-09-20T22:18:39.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:18:38 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi037.lykblh/mirror_snapshot_schedule"}]: dispatch 2024-09-20T22:18:39.792 INFO:teuthology.orchestra.run.smithi037.stdout: 5 20400 10.97 MB/sec warmup 57 sec latency 41.494 ms 2024-09-20T22:18:39.870 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:18:39 smithi037 ceph-mon[23718]: pgmap v267: 97 pgs: 97 active+clean; 175 MiB data, 654 MiB used, 715 GiB / 715 GiB avail; 6.8 KiB/s rd, 6.6 MiB/s wr, 618 op/s 2024-09-20T22:18:40.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:18:39 smithi194 ceph-mon[25957]: pgmap v267: 97 pgs: 97 active+clean; 175 MiB data, 654 MiB used, 715 GiB / 715 GiB avail; 6.8 KiB/s rd, 6.6 MiB/s wr, 618 op/s 2024-09-20T22:18:40.374 INFO:teuthology.orchestra.run.smithi037.stdout:true 2024-09-20T22:18:40.792 INFO:teuthology.orchestra.run.smithi037.stdout: 5 20953 11.20 MB/sec warmup 58 sec latency 44.191 ms 2024-09-20T22:18:41.235 INFO:teuthology.orchestra.run.smithi037.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T22:18:41.235 INFO:teuthology.orchestra.run.smithi037.stdout:alertmanager.smithi037 smithi037 *:9093,9094 running (6m) 60s ago 7m 24.3M - ba2b418f427c bb407b78f0c9 2024-09-20T22:18:41.235 INFO:teuthology.orchestra.run.smithi037.stdout:crash.smithi037 smithi037 running (7m) 60s ago 7m 6400k - 17.2.0 e1d6a67b021e 88301022bdc8 2024-09-20T22:18:41.235 INFO:teuthology.orchestra.run.smithi037.stdout:crash.smithi194 smithi194 running (6m) 63s ago 6m 6405k - 17.2.0 e1d6a67b021e b553601d7ca5 2024-09-20T22:18:41.236 INFO:teuthology.orchestra.run.smithi037.stdout:grafana.smithi037 smithi037 *:3000 running (5m) 60s ago 7m 52.0M - 8.3.5 dad864ee21e9 3ccb7bb6f7ae 2024-09-20T22:18:41.236 INFO:teuthology.orchestra.run.smithi037.stdout:mds.foofs.smithi037.sdtedi smithi037 running (93s) 60s ago 91s 13.2M - 17.2.0 e1d6a67b021e 0a7898d25b8d 2024-09-20T22:18:41.236 INFO:teuthology.orchestra.run.smithi037.stdout:mds.foofs.smithi194.lkroso smithi194 running (89s) 63s ago 88s 13.4M - 17.2.0 e1d6a67b021e 17736de30008 2024-09-20T22:18:41.236 INFO:teuthology.orchestra.run.smithi037.stdout:mgr.smithi037.lykblh smithi037 *:9283 running (8m) 60s ago 8m 460M - 17.2.0 e1d6a67b021e 3ef1f99a6ba9 2024-09-20T22:18:41.236 INFO:teuthology.orchestra.run.smithi037.stdout:mgr.smithi194.bkcglt smithi194 *:8443,9283 running (6m) 63s ago 6m 413M - 17.2.0 e1d6a67b021e 7cc5497632b6 2024-09-20T22:18:41.236 INFO:teuthology.orchestra.run.smithi037.stdout:mon.smithi037 smithi037 running (8m) 60s ago 8m 43.5M 2048M 17.2.0 e1d6a67b021e 71f591122786 2024-09-20T22:18:41.236 INFO:teuthology.orchestra.run.smithi037.stdout:mon.smithi194 smithi194 running (6m) 63s ago 6m 37.9M 2048M 17.2.0 e1d6a67b021e 0b8814ae6c66 2024-09-20T22:18:41.236 INFO:teuthology.orchestra.run.smithi037.stdout:nfs.foo.0.0.smithi037.spabdc smithi037 *:2049 running (71s) 60s ago 71s 51.1M - 4.0 e1d6a67b021e 858ff9d558a2 2024-09-20T22:18:41.236 INFO:teuthology.orchestra.run.smithi037.stdout:nfs.foo.1.0.smithi194.qzjyej smithi194 *:2049 running (66s) 63s ago 65s 52.8M - 4.0 e1d6a67b021e c07fe0eb7c75 2024-09-20T22:18:41.236 INFO:teuthology.orchestra.run.smithi037.stdout:node-exporter.smithi037 smithi037 *:9100 running (7m) 60s ago 7m 16.5M - 1dbe0e931976 1f92060f034e 2024-09-20T22:18:41.236 INFO:teuthology.orchestra.run.smithi037.stdout:node-exporter.smithi194 smithi194 *:9100 running (6m) 63s ago 6m 17.7M - 1dbe0e931976 3a80a2ef6678 2024-09-20T22:18:41.236 INFO:teuthology.orchestra.run.smithi037.stdout:osd.0 smithi037 running (5m) 60s ago 5m 43.2M 3217M 17.2.0 e1d6a67b021e e54e84fd5030 2024-09-20T22:18:41.237 INFO:teuthology.orchestra.run.smithi037.stdout:osd.1 smithi037 running (5m) 60s ago 5m 42.7M 3217M 17.2.0 e1d6a67b021e 09647c133032 2024-09-20T22:18:41.237 INFO:teuthology.orchestra.run.smithi037.stdout:osd.2 smithi037 running (4m) 60s ago 4m 42.2M 3217M 17.2.0 e1d6a67b021e 6c15b1175256 2024-09-20T22:18:41.237 INFO:teuthology.orchestra.run.smithi037.stdout:osd.3 smithi037 running (4m) 60s ago 4m 39.6M 3217M 17.2.0 e1d6a67b021e 3a22952c6475 2024-09-20T22:18:41.237 INFO:teuthology.orchestra.run.smithi037.stdout:osd.4 smithi194 running (4m) 63s ago 4m 39.2M 3984M 17.2.0 e1d6a67b021e fa107be8915c 2024-09-20T22:18:41.237 INFO:teuthology.orchestra.run.smithi037.stdout:osd.5 smithi194 running (3m) 63s ago 3m 39.1M 3984M 17.2.0 e1d6a67b021e 290b21cbd131 2024-09-20T22:18:41.237 INFO:teuthology.orchestra.run.smithi037.stdout:osd.6 smithi194 running (3m) 63s ago 3m 41.7M 3984M 17.2.0 e1d6a67b021e 27f0720639ed 2024-09-20T22:18:41.237 INFO:teuthology.orchestra.run.smithi037.stdout:osd.7 smithi194 running (2m) 63s ago 2m 41.0M 3984M 17.2.0 e1d6a67b021e 62d163f4fd92 2024-09-20T22:18:41.237 INFO:teuthology.orchestra.run.smithi037.stdout:prometheus.smithi037 smithi037 *:9095 running (6m) 60s ago 6m 61.8M - 514e6a882f6e 32020a1cc2f4 2024-09-20T22:18:41.679 INFO:teuthology.orchestra.run.smithi037.stdout:{ 2024-09-20T22:18:41.679 INFO:teuthology.orchestra.run.smithi037.stdout: "mon": { 2024-09-20T22:18:41.679 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T22:18:41.679 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:18:41.679 INFO:teuthology.orchestra.run.smithi037.stdout: "mgr": { 2024-09-20T22:18:41.679 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T22:18:41.680 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:18:41.680 INFO:teuthology.orchestra.run.smithi037.stdout: "osd": { 2024-09-20T22:18:41.680 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-20T22:18:41.680 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:18:41.680 INFO:teuthology.orchestra.run.smithi037.stdout: "mds": { 2024-09-20T22:18:41.680 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T22:18:41.680 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:18:41.680 INFO:teuthology.orchestra.run.smithi037.stdout: "overall": { 2024-09-20T22:18:41.680 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-09-20T22:18:41.680 INFO:teuthology.orchestra.run.smithi037.stdout: } 2024-09-20T22:18:41.680 INFO:teuthology.orchestra.run.smithi037.stdout:} 2024-09-20T22:18:41.792 INFO:teuthology.orchestra.run.smithi037.stdout: 5 21243 11.21 MB/sec warmup 59 sec latency 71.067 ms 2024-09-20T22:18:41.888 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:18:41 smithi037 ceph-mon[23718]: pgmap v268: 97 pgs: 97 active+clean; 175 MiB data, 674 MiB used, 715 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.5 MiB/s wr, 424 op/s 2024-09-20T22:18:41.888 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:18:41 smithi037 ceph-mon[23718]: from='client.14640 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:18:41.888 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:18:41 smithi037 ceph-mon[23718]: from='client.14644 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:18:41.888 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:18:41 smithi037 ceph-mon[23718]: from='client.? 172.21.15.37:0/365819468' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:18:42.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:18:41 smithi194 ceph-mon[25957]: pgmap v268: 97 pgs: 97 active+clean; 175 MiB data, 674 MiB used, 715 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.5 MiB/s wr, 424 op/s 2024-09-20T22:18:42.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:18:41 smithi194 ceph-mon[25957]: from='client.14640 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:18:42.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:18:41 smithi194 ceph-mon[25957]: from='client.14644 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:18:42.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:18:41 smithi194 ceph-mon[25957]: from='client.? 172.21.15.37:0/365819468' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:18:42.193 INFO:teuthology.orchestra.run.smithi037.stdout:{ 2024-09-20T22:18:42.193 INFO:teuthology.orchestra.run.smithi037.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5", 2024-09-20T22:18:42.193 INFO:teuthology.orchestra.run.smithi037.stdout: "in_progress": true, 2024-09-20T22:18:42.193 INFO:teuthology.orchestra.run.smithi037.stdout: "services_complete": [], 2024-09-20T22:18:42.193 INFO:teuthology.orchestra.run.smithi037.stdout: "progress": "0/23 daemons upgraded", 2024-09-20T22:18:42.193 INFO:teuthology.orchestra.run.smithi037.stdout: "message": "Pulling quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5 image on host smithi194" 2024-09-20T22:18:42.193 INFO:teuthology.orchestra.run.smithi037.stdout:} 2024-09-20T22:18:42.792 INFO:teuthology.orchestra.run.smithi037.stdout: 5 21835 11.31 MB/sec warmup 60 sec latency 33.832 ms 2024-09-20T22:18:42.841 INFO:teuthology.orchestra.run.smithi037.stdout:HEALTH_OK 2024-09-20T22:18:42.893 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:18:42 smithi037 ceph-mon[23718]: from='client.14648 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:18:43.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:18:42 smithi194 ceph-mon[25957]: from='client.14648 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:18:43.792 INFO:teuthology.orchestra.run.smithi037.stdout: 5 22535 11.34 MB/sec warmup 61 sec latency 16.548 ms 2024-09-20T22:18:43.899 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:18:43 smithi037 ceph-mon[23718]: from='client.14656 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:18:43.899 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:18:43 smithi037 ceph-mon[23718]: pgmap v269: 97 pgs: 97 active+clean; 183 MiB data, 701 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 5.4 MiB/s wr, 587 op/s 2024-09-20T22:18:43.899 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:18:43 smithi037 ceph-mon[23718]: from='client.? 172.21.15.37:0/3838385729' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T22:18:44.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:18:43 smithi194 ceph-mon[25957]: from='client.14656 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:18:44.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:18:43 smithi194 ceph-mon[25957]: pgmap v269: 97 pgs: 97 active+clean; 183 MiB data, 701 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 5.4 MiB/s wr, 587 op/s 2024-09-20T22:18:44.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:18:43 smithi194 ceph-mon[25957]: from='client.? 172.21.15.37:0/3838385729' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T22:18:44.792 INFO:teuthology.orchestra.run.smithi037.stdout: 5 23438 11.42 MB/sec warmup 62 sec latency 22.791 ms 2024-09-20T22:18:45.792 INFO:teuthology.orchestra.run.smithi037.stdout: 5 24510 11.84 MB/sec warmup 63 sec latency 11.182 ms 2024-09-20T22:18:45.914 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:18:45 smithi037 ceph-mon[23718]: pgmap v270: 97 pgs: 97 active+clean; 192 MiB data, 736 MiB used, 715 GiB / 715 GiB avail; 7.0 KiB/s rd, 7.3 MiB/s wr, 698 op/s 2024-09-20T22:18:46.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:18:45 smithi194 ceph-mon[25957]: pgmap v270: 97 pgs: 97 active+clean; 192 MiB data, 736 MiB used, 715 GiB / 715 GiB avail; 7.0 KiB/s rd, 7.3 MiB/s wr, 698 op/s 2024-09-20T22:18:46.792 INFO:teuthology.orchestra.run.smithi037.stdout: 5 25192 12.07 MB/sec warmup 64 sec latency 233.046 ms 2024-09-20T22:18:47.792 INFO:teuthology.orchestra.run.smithi037.stdout: 5 25883 12.01 MB/sec warmup 65 sec latency 43.973 ms 2024-09-20T22:18:47.914 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:18:47 smithi037 ceph-mon[23718]: pgmap v271: 97 pgs: 97 active+clean; 193 MiB data, 757 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 6.0 MiB/s wr, 616 op/s 2024-09-20T22:18:48.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:18:47 smithi194 ceph-mon[25957]: pgmap v271: 97 pgs: 97 active+clean; 193 MiB data, 757 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 6.0 MiB/s wr, 616 op/s 2024-09-20T22:18:48.792 INFO:teuthology.orchestra.run.smithi037.stdout: 5 26822 12.16 MB/sec warmup 66 sec latency 34.285 ms 2024-09-20T22:18:49.792 INFO:teuthology.orchestra.run.smithi037.stdout: 5 27435 12.21 MB/sec warmup 67 sec latency 42.745 ms 2024-09-20T22:18:50.002 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:18:49 smithi037 ceph-mon[23718]: pgmap v272: 97 pgs: 97 active+clean; 203 MiB data, 831 MiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 9.4 MiB/s wr, 843 op/s 2024-09-20T22:18:50.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:18:49 smithi194 ceph-mon[25957]: pgmap v272: 97 pgs: 97 active+clean; 203 MiB data, 831 MiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 9.4 MiB/s wr, 843 op/s 2024-09-20T22:18:50.792 INFO:teuthology.orchestra.run.smithi037.stdout: 5 28068 12.42 MB/sec warmup 68 sec latency 85.044 ms 2024-09-20T22:18:51.792 INFO:teuthology.orchestra.run.smithi037.stdout: 5 28607 12.54 MB/sec warmup 69 sec latency 59.480 ms 2024-09-20T22:18:52.043 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:18:51 smithi037 ceph-mon[23718]: pgmap v273: 97 pgs: 97 active+clean; 210 MiB data, 849 MiB used, 714 GiB / 715 GiB avail; 9.9 KiB/s rd, 8.8 MiB/s wr, 867 op/s 2024-09-20T22:18:52.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:18:51 smithi194 ceph-mon[25957]: pgmap v273: 97 pgs: 97 active+clean; 210 MiB data, 849 MiB used, 714 GiB / 715 GiB avail; 9.9 KiB/s rd, 8.8 MiB/s wr, 867 op/s 2024-09-20T22:18:52.792 INFO:teuthology.orchestra.run.smithi037.stdout: 5 28755 12.41 MB/sec warmup 70 sec latency 51.267 ms 2024-09-20T22:18:53.792 INFO:teuthology.orchestra.run.smithi037.stdout: 5 28975 12.26 MB/sec warmup 71 sec latency 74.047 ms 2024-09-20T22:18:53.850 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:18:53 smithi037 ceph-mon[23718]: pgmap v274: 97 pgs: 97 active+clean; 210 MiB data, 862 MiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 8.8 MiB/s wr, 871 op/s 2024-09-20T22:18:54.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:18:53 smithi194 ceph-mon[25957]: pgmap v274: 97 pgs: 97 active+clean; 210 MiB data, 862 MiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 8.8 MiB/s wr, 871 op/s 2024-09-20T22:18:54.792 INFO:teuthology.orchestra.run.smithi037.stdout: 5 29515 12.23 MB/sec warmup 72 sec latency 64.157 ms 2024-09-20T22:18:55.792 INFO:teuthology.orchestra.run.smithi037.stdout: 5 30338 12.30 MB/sec warmup 73 sec latency 22.813 ms 2024-09-20T22:18:55.888 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:18:55 smithi037 ceph-mon[23718]: pgmap v275: 97 pgs: 97 active+clean; 215 MiB data, 904 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 11 MiB/s wr, 887 op/s 2024-09-20T22:18:56.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:18:55 smithi194 ceph-mon[25957]: pgmap v275: 97 pgs: 97 active+clean; 215 MiB data, 904 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 11 MiB/s wr, 887 op/s 2024-09-20T22:18:56.792 INFO:teuthology.orchestra.run.smithi037.stdout: 5 30785 12.22 MB/sec warmup 74 sec latency 50.919 ms 2024-09-20T22:18:57.792 INFO:teuthology.orchestra.run.smithi037.stdout: 5 31088 12.22 MB/sec warmup 75 sec latency 63.910 ms 2024-09-20T22:18:57.978 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:18:57 smithi037 ceph-mon[23718]: pgmap v276: 97 pgs: 97 active+clean; 219 MiB data, 924 MiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 8.9 MiB/s wr, 832 op/s 2024-09-20T22:18:58.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:18:57 smithi194 ceph-mon[25957]: pgmap v276: 97 pgs: 97 active+clean; 219 MiB data, 924 MiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 8.9 MiB/s wr, 832 op/s 2024-09-20T22:18:58.793 INFO:teuthology.orchestra.run.smithi037.stdout: 5 31460 12.27 MB/sec warmup 76 sec latency 66.583 ms 2024-09-20T22:18:59.792 INFO:teuthology.orchestra.run.smithi037.stdout: 5 32000 12.39 MB/sec warmup 77 sec latency 74.782 ms 2024-09-20T22:19:00.026 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:18:59 smithi037 ceph-mon[23718]: pgmap v277: 97 pgs: 97 active+clean; 215 MiB data, 968 MiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 10 MiB/s wr, 995 op/s 2024-09-20T22:19:00.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:18:59 smithi194 ceph-mon[25957]: pgmap v277: 97 pgs: 97 active+clean; 215 MiB data, 968 MiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 10 MiB/s wr, 995 op/s 2024-09-20T22:19:00.793 INFO:teuthology.orchestra.run.smithi037.stdout: 5 32685 12.40 MB/sec warmup 78 sec latency 20.065 ms 2024-09-20T22:19:01.793 INFO:teuthology.orchestra.run.smithi037.stdout: 5 33353 12.43 MB/sec warmup 79 sec latency 21.368 ms 2024-09-20T22:19:01.880 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:01 smithi037 ceph-mon[23718]: pgmap v278: 97 pgs: 97 active+clean; 216 MiB data, 984 MiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 6.6 MiB/s wr, 679 op/s 2024-09-20T22:19:02.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:01 smithi194 ceph-mon[25957]: pgmap v278: 97 pgs: 97 active+clean; 216 MiB data, 984 MiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 6.6 MiB/s wr, 679 op/s 2024-09-20T22:19:02.793 INFO:teuthology.orchestra.run.smithi037.stdout: 5 34219 12.48 MB/sec warmup 80 sec latency 44.928 ms 2024-09-20T22:19:03.793 INFO:teuthology.orchestra.run.smithi037.stdout: 5 34504 12.43 MB/sec warmup 81 sec latency 59.073 ms 2024-09-20T22:19:03.925 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:03 smithi037 ceph-mon[23718]: pgmap v279: 97 pgs: 97 active+clean; 222 MiB data, 998 MiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 6.1 MiB/s wr, 590 op/s 2024-09-20T22:19:04.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:03 smithi194 ceph-mon[25957]: pgmap v279: 97 pgs: 97 active+clean; 222 MiB data, 998 MiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 6.1 MiB/s wr, 590 op/s 2024-09-20T22:19:04.793 INFO:teuthology.orchestra.run.smithi037.stdout: 5 34823 12.44 MB/sec warmup 82 sec latency 60.189 ms 2024-09-20T22:19:05.793 INFO:teuthology.orchestra.run.smithi037.stdout: 5 35190 12.49 MB/sec warmup 83 sec latency 124.040 ms 2024-09-20T22:19:05.965 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:05 smithi037 ceph-mon[23718]: pgmap v280: 97 pgs: 97 active+clean; 224 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 7.2 KiB/s rd, 8.4 MiB/s wr, 732 op/s 2024-09-20T22:19:06.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:05 smithi194 ceph-mon[25957]: pgmap v280: 97 pgs: 97 active+clean; 224 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 7.2 KiB/s rd, 8.4 MiB/s wr, 732 op/s 2024-09-20T22:19:06.793 INFO:teuthology.orchestra.run.smithi037.stdout: 5 35795 12.59 MB/sec warmup 84 sec latency 36.479 ms 2024-09-20T22:19:07.473 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:07 smithi037 ceph-mon[23718]: pgmap v281: 97 pgs: 97 active+clean; 230 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 6.5 MiB/s wr, 709 op/s 2024-09-20T22:19:07.473 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:07 smithi037 ceph-mon[23718]: Upgrade: Updating mgr.smithi194.bkcglt 2024-09-20T22:19:07.473 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:07 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:19:07.473 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:07 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi194.bkcglt", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-20T22:19:07.473 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:07 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-20T22:19:07.473 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:07 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:19:07.473 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:07 smithi037 ceph-mon[23718]: Deploying daemon mgr.smithi194.bkcglt on smithi194 2024-09-20T22:19:07.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:07 smithi194 ceph-mon[25957]: pgmap v281: 97 pgs: 97 active+clean; 230 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 6.5 MiB/s wr, 709 op/s 2024-09-20T22:19:07.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:07 smithi194 ceph-mon[25957]: Upgrade: Updating mgr.smithi194.bkcglt 2024-09-20T22:19:07.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:07 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:19:07.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:07 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi194.bkcglt", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-20T22:19:07.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:07 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-20T22:19:07.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:07 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:19:07.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:07 smithi194 ceph-mon[25957]: Deploying daemon mgr.smithi194.bkcglt on smithi194 2024-09-20T22:19:07.793 INFO:teuthology.orchestra.run.smithi037.stdout: 5 36366 12.54 MB/sec warmup 85 sec latency 54.677 ms 2024-09-20T22:19:08.793 INFO:teuthology.orchestra.run.smithi037.stdout: 5 37103 12.58 MB/sec warmup 86 sec latency 32.789 ms 2024-09-20T22:19:09.481 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:09 smithi037 ceph-mon[23718]: pgmap v282: 97 pgs: 97 active+clean; 238 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 8.6 KiB/s rd, 8.3 MiB/s wr, 721 op/s 2024-09-20T22:19:09.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:09 smithi194 ceph-mon[25957]: pgmap v282: 97 pgs: 97 active+clean; 238 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 8.6 KiB/s rd, 8.3 MiB/s wr, 721 op/s 2024-09-20T22:19:09.793 INFO:teuthology.orchestra.run.smithi037.stdout: 5 37833 12.62 MB/sec warmup 87 sec latency 20.471 ms 2024-09-20T22:19:10.793 INFO:teuthology.orchestra.run.smithi037.stdout: 5 38541 12.75 MB/sec warmup 88 sec latency 31.733 ms 2024-09-20T22:19:11.672 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:11 smithi194 ceph-mon[25957]: pgmap v283: 97 pgs: 97 active+clean; 244 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 8.4 KiB/s rd, 7.2 MiB/s wr, 671 op/s 2024-09-20T22:19:11.672 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:11 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:19:11.672 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:11 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:19:11.672 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:11 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:19:11.672 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:11 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:19:11.740 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:11 smithi037 ceph-mon[23718]: pgmap v283: 97 pgs: 97 active+clean; 244 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 8.4 KiB/s rd, 7.2 MiB/s wr, 671 op/s 2024-09-20T22:19:11.740 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:11 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:19:11.740 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:11 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:19:11.740 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:11 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:19:11.741 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:11 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:19:11.793 INFO:teuthology.orchestra.run.smithi037.stdout: 5 39193 12.89 MB/sec warmup 89 sec latency 26.308 ms 2024-09-20T22:19:12.793 INFO:teuthology.orchestra.run.smithi037.stdout: 5 39872 12.90 MB/sec warmup 90 sec latency 21.688 ms 2024-09-20T22:19:13.406 INFO:teuthology.orchestra.run.smithi037.stdout:true 2024-09-20T22:19:13.755 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:13 smithi037 ceph-mon[23718]: pgmap v284: 97 pgs: 97 active+clean; 250 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 7.9 MiB/s wr, 823 op/s 2024-09-20T22:19:13.793 INFO:teuthology.orchestra.run.smithi037.stdout: 5 40649 12.94 MB/sec warmup 91 sec latency 21.645 ms 2024-09-20T22:19:13.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:13 smithi194 ceph-mon[25957]: pgmap v284: 97 pgs: 97 active+clean; 250 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 7.9 MiB/s wr, 823 op/s 2024-09-20T22:19:14.482 INFO:teuthology.orchestra.run.smithi037.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T22:19:14.482 INFO:teuthology.orchestra.run.smithi037.stdout:alertmanager.smithi037 smithi037 *:9093,9094 running (6m) 94s ago 8m 24.3M - ba2b418f427c bb407b78f0c9 2024-09-20T22:19:14.482 INFO:teuthology.orchestra.run.smithi037.stdout:crash.smithi037 smithi037 running (7m) 94s ago 7m 6400k - 17.2.0 e1d6a67b021e 88301022bdc8 2024-09-20T22:19:14.482 INFO:teuthology.orchestra.run.smithi037.stdout:crash.smithi194 smithi194 running (7m) 96s ago 7m 6405k - 17.2.0 e1d6a67b021e b553601d7ca5 2024-09-20T22:19:14.482 INFO:teuthology.orchestra.run.smithi037.stdout:grafana.smithi037 smithi037 *:3000 running (6m) 94s ago 7m 52.0M - 8.3.5 dad864ee21e9 3ccb7bb6f7ae 2024-09-20T22:19:14.483 INFO:teuthology.orchestra.run.smithi037.stdout:mds.foofs.smithi037.sdtedi smithi037 running (2m) 94s ago 2m 13.2M - 17.2.0 e1d6a67b021e 0a7898d25b8d 2024-09-20T22:19:14.483 INFO:teuthology.orchestra.run.smithi037.stdout:mds.foofs.smithi194.lkroso smithi194 running (2m) 96s ago 2m 13.4M - 17.2.0 e1d6a67b021e 17736de30008 2024-09-20T22:19:14.483 INFO:teuthology.orchestra.run.smithi037.stdout:mgr.smithi037.lykblh smithi037 *:9283 running (9m) 94s ago 9m 460M - 17.2.0 e1d6a67b021e 3ef1f99a6ba9 2024-09-20T22:19:14.483 INFO:teuthology.orchestra.run.smithi037.stdout:mgr.smithi194.bkcglt smithi194 *:8443,9283 starting - - - - 2024-09-20T22:19:14.483 INFO:teuthology.orchestra.run.smithi037.stdout:mon.smithi037 smithi037 running (9m) 94s ago 9m 43.5M 2048M 17.2.0 e1d6a67b021e 71f591122786 2024-09-20T22:19:14.483 INFO:teuthology.orchestra.run.smithi037.stdout:mon.smithi194 smithi194 running (7m) 96s ago 7m 37.9M 2048M 17.2.0 e1d6a67b021e 0b8814ae6c66 2024-09-20T22:19:14.483 INFO:teuthology.orchestra.run.smithi037.stdout:nfs.foo.0.0.smithi037.spabdc smithi037 *:2049 running (104s) 94s ago 104s 51.1M - 4.0 e1d6a67b021e 858ff9d558a2 2024-09-20T22:19:14.483 INFO:teuthology.orchestra.run.smithi037.stdout:nfs.foo.1.0.smithi194.qzjyej smithi194 *:2049 running (99s) 96s ago 99s 52.8M - 4.0 e1d6a67b021e c07fe0eb7c75 2024-09-20T22:19:14.483 INFO:teuthology.orchestra.run.smithi037.stdout:node-exporter.smithi037 smithi037 *:9100 running (7m) 94s ago 7m 16.5M - 1dbe0e931976 1f92060f034e 2024-09-20T22:19:14.483 INFO:teuthology.orchestra.run.smithi037.stdout:node-exporter.smithi194 smithi194 *:9100 running (7m) 96s ago 7m 17.7M - 1dbe0e931976 3a80a2ef6678 2024-09-20T22:19:14.483 INFO:teuthology.orchestra.run.smithi037.stdout:osd.0 smithi037 running (6m) 94s ago 6m 43.2M 3217M 17.2.0 e1d6a67b021e e54e84fd5030 2024-09-20T22:19:14.483 INFO:teuthology.orchestra.run.smithi037.stdout:osd.1 smithi037 running (6m) 94s ago 6m 42.7M 3217M 17.2.0 e1d6a67b021e 09647c133032 2024-09-20T22:19:14.483 INFO:teuthology.orchestra.run.smithi037.stdout:osd.2 smithi037 running (5m) 94s ago 5m 42.2M 3217M 17.2.0 e1d6a67b021e 6c15b1175256 2024-09-20T22:19:14.483 INFO:teuthology.orchestra.run.smithi037.stdout:osd.3 smithi037 running (5m) 94s ago 5m 39.6M 3217M 17.2.0 e1d6a67b021e 3a22952c6475 2024-09-20T22:19:14.484 INFO:teuthology.orchestra.run.smithi037.stdout:osd.4 smithi194 running (4m) 96s ago 4m 39.2M 3984M 17.2.0 e1d6a67b021e fa107be8915c 2024-09-20T22:19:14.484 INFO:teuthology.orchestra.run.smithi037.stdout:osd.5 smithi194 running (4m) 96s ago 4m 39.1M 3984M 17.2.0 e1d6a67b021e 290b21cbd131 2024-09-20T22:19:14.484 INFO:teuthology.orchestra.run.smithi037.stdout:osd.6 smithi194 running (3m) 96s ago 3m 41.7M 3984M 17.2.0 e1d6a67b021e 27f0720639ed 2024-09-20T22:19:14.484 INFO:teuthology.orchestra.run.smithi037.stdout:osd.7 smithi194 running (3m) 96s ago 3m 41.0M 3984M 17.2.0 e1d6a67b021e 62d163f4fd92 2024-09-20T22:19:14.484 INFO:teuthology.orchestra.run.smithi037.stdout:prometheus.smithi037 smithi037 *:9095 running (6m) 94s ago 6m 61.8M - 514e6a882f6e 32020a1cc2f4 2024-09-20T22:19:14.760 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:14 smithi037 ceph-mon[23718]: from='client.14666 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:19:14.793 INFO:teuthology.orchestra.run.smithi037.stdout: 5 41384 12.97 MB/sec warmup 92 sec latency 22.744 ms 2024-09-20T22:19:14.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:14 smithi194 ceph-mon[25957]: from='client.14666 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:19:15.105 INFO:teuthology.orchestra.run.smithi037.stdout:{ 2024-09-20T22:19:15.105 INFO:teuthology.orchestra.run.smithi037.stdout: "mon": { 2024-09-20T22:19:15.105 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T22:19:15.105 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:19:15.105 INFO:teuthology.orchestra.run.smithi037.stdout: "mgr": { 2024-09-20T22:19:15.105 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T22:19:15.105 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:19:15.105 INFO:teuthology.orchestra.run.smithi037.stdout: "osd": { 2024-09-20T22:19:15.105 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-20T22:19:15.105 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:19:15.105 INFO:teuthology.orchestra.run.smithi037.stdout: "mds": { 2024-09-20T22:19:15.106 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T22:19:15.106 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:19:15.106 INFO:teuthology.orchestra.run.smithi037.stdout: "overall": { 2024-09-20T22:19:15.106 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-09-20T22:19:15.106 INFO:teuthology.orchestra.run.smithi037.stdout: } 2024-09-20T22:19:15.106 INFO:teuthology.orchestra.run.smithi037.stdout:} 2024-09-20T22:19:15.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:15 smithi194 ceph-mon[25957]: from='client.14670 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:19:15.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:15 smithi194 ceph-mon[25957]: pgmap v285: 97 pgs: 97 active+clean; 255 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 10 MiB/s wr, 928 op/s 2024-09-20T22:19:15.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:15 smithi194 ceph-mon[25957]: from='client.14674 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:19:15.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:15 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:19:15.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:15 smithi194 ceph-mon[25957]: from='client.? 172.21.15.37:0/752658184' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:19:15.636 INFO:teuthology.orchestra.run.smithi037.stdout:{ 2024-09-20T22:19:15.636 INFO:teuthology.orchestra.run.smithi037.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5", 2024-09-20T22:19:15.636 INFO:teuthology.orchestra.run.smithi037.stdout: "in_progress": true, 2024-09-20T22:19:15.636 INFO:teuthology.orchestra.run.smithi037.stdout: "services_complete": [], 2024-09-20T22:19:15.636 INFO:teuthology.orchestra.run.smithi037.stdout: "progress": "1/23 daemons upgraded", 2024-09-20T22:19:15.636 INFO:teuthology.orchestra.run.smithi037.stdout: "message": "Currently upgrading mgr daemons" 2024-09-20T22:19:15.636 INFO:teuthology.orchestra.run.smithi037.stdout:} 2024-09-20T22:19:15.764 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:15 smithi037 ceph-mon[23718]: from='client.14670 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:19:15.764 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:15 smithi037 ceph-mon[23718]: pgmap v285: 97 pgs: 97 active+clean; 255 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 10 MiB/s wr, 928 op/s 2024-09-20T22:19:15.764 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:15 smithi037 ceph-mon[23718]: from='client.14674 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:19:15.764 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:15 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:19:15.764 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:15 smithi037 ceph-mon[23718]: from='client.? 172.21.15.37:0/752658184' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:19:15.793 INFO:teuthology.orchestra.run.smithi037.stdout: 5 42032 13.08 MB/sec warmup 93 sec latency 24.290 ms 2024-09-20T22:19:16.282 INFO:teuthology.orchestra.run.smithi037.stdout:HEALTH_OK 2024-09-20T22:19:16.768 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:16 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:19:16.768 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:16 smithi037 ceph-mon[23718]: from='client.14680 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:19:16.768 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:16 smithi037 ceph-mon[23718]: from='client.? 172.21.15.37:0/2976604572' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T22:19:16.793 INFO:teuthology.orchestra.run.smithi037.stdout: 5 42690 13.21 MB/sec warmup 94 sec latency 27.156 ms 2024-09-20T22:19:16.999 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:16 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:19:16.999 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:16 smithi194 ceph-mon[25957]: from='client.14680 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:19:16.999 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:16 smithi194 ceph-mon[25957]: from='client.? 172.21.15.37:0/2976604572' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T22:19:17.774 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:17 smithi037 ceph-mon[23718]: pgmap v286: 97 pgs: 97 active+clean; 259 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 8.5 MiB/s wr, 865 op/s 2024-09-20T22:19:17.774 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:17 smithi037 ceph-mon[23718]: Standby manager daemon smithi194.bkcglt restarted 2024-09-20T22:19:17.775 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:17 smithi037 ceph-mon[23718]: Standby manager daemon smithi194.bkcglt started 2024-09-20T22:19:17.775 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:17 smithi037 ceph-mon[23718]: from='mgr.? 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi194.bkcglt/crt"}]: dispatch 2024-09-20T22:19:17.775 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:17 smithi037 ceph-mon[23718]: from='mgr.? 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-20T22:19:17.775 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:17 smithi037 ceph-mon[23718]: from='mgr.? 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi194.bkcglt/key"}]: dispatch 2024-09-20T22:19:17.775 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:17 smithi037 ceph-mon[23718]: from='mgr.? 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-20T22:19:17.793 INFO:teuthology.orchestra.run.smithi037.stdout: 5 43297 13.21 MB/sec warmup 95 sec latency 22.137 ms 2024-09-20T22:19:18.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:17 smithi194 ceph-mon[25957]: pgmap v286: 97 pgs: 97 active+clean; 259 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 8.5 MiB/s wr, 865 op/s 2024-09-20T22:19:18.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:17 smithi194 ceph-mon[25957]: Standby manager daemon smithi194.bkcglt restarted 2024-09-20T22:19:18.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:17 smithi194 ceph-mon[25957]: Standby manager daemon smithi194.bkcglt started 2024-09-20T22:19:18.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:17 smithi194 ceph-mon[25957]: from='mgr.? 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi194.bkcglt/crt"}]: dispatch 2024-09-20T22:19:18.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:17 smithi194 ceph-mon[25957]: from='mgr.? 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-20T22:19:18.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:17 smithi194 ceph-mon[25957]: from='mgr.? 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi194.bkcglt/key"}]: dispatch 2024-09-20T22:19:18.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:17 smithi194 ceph-mon[25957]: from='mgr.? 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-20T22:19:18.793 INFO:teuthology.orchestra.run.smithi037.stdout: 5 44017 13.23 MB/sec warmup 96 sec latency 30.400 ms 2024-09-20T22:19:18.828 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:18 smithi037 ceph-mon[23718]: mgrmap e22: smithi037.lykblh(active, since 7m), standbys: smithi194.bkcglt 2024-09-20T22:19:19.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:18 smithi194 ceph-mon[25957]: mgrmap e22: smithi037.lykblh(active, since 7m), standbys: smithi194.bkcglt 2024-09-20T22:19:19.793 INFO:teuthology.orchestra.run.smithi037.stdout: 5 44390 13.14 MB/sec warmup 97 sec latency 43.055 ms 2024-09-20T22:19:19.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:19 smithi194 ceph-mon[25957]: pgmap v287: 97 pgs: 97 active+clean; 266 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 11 MiB/s wr, 977 op/s 2024-09-20T22:19:19.849 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:19 smithi037 ceph-mon[23718]: pgmap v287: 97 pgs: 97 active+clean; 266 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 11 MiB/s wr, 977 op/s 2024-09-20T22:19:20.793 INFO:teuthology.orchestra.run.smithi037.stdout: 5 44745 13.11 MB/sec warmup 98 sec latency 85.322 ms 2024-09-20T22:19:21.701 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:21 smithi037 ceph-mon[23718]: pgmap v288: 97 pgs: 97 active+clean; 266 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.0 MiB/s wr, 919 op/s 2024-09-20T22:19:21.793 INFO:teuthology.orchestra.run.smithi037.stdout: 5 45271 13.14 MB/sec warmup 99 sec latency 49.327 ms 2024-09-20T22:19:22.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:21 smithi194 ceph-mon[25957]: pgmap v288: 97 pgs: 97 active+clean; 266 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.0 MiB/s wr, 919 op/s 2024-09-20T22:19:22.793 INFO:teuthology.orchestra.run.smithi037.stdout: 5 45593 13.13 MB/sec warmup 100 sec latency 66.692 ms 2024-09-20T22:19:23.750 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:23 smithi037 ceph-mon[23718]: pgmap v289: 97 pgs: 97 active+clean; 268 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 8.8 MiB/s wr, 885 op/s 2024-09-20T22:19:23.793 INFO:teuthology.orchestra.run.smithi037.stdout: 5 46036 13.19 MB/sec warmup 101 sec latency 43.542 ms 2024-09-20T22:19:24.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:23 smithi194 ceph-mon[25957]: pgmap v289: 97 pgs: 97 active+clean; 268 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 8.8 MiB/s wr, 885 op/s 2024-09-20T22:19:24.793 INFO:teuthology.orchestra.run.smithi037.stdout: 5 46551 13.20 MB/sec warmup 102 sec latency 111.348 ms 2024-09-20T22:19:25.770 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:25 smithi037 ceph-mon[23718]: pgmap v290: 97 pgs: 97 active+clean; 267 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.7 MiB/s wr, 808 op/s 2024-09-20T22:19:25.794 INFO:teuthology.orchestra.run.smithi037.stdout: 5 47223 13.18 MB/sec warmup 103 sec latency 98.135 ms 2024-09-20T22:19:26.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:25 smithi194 ceph-mon[25957]: pgmap v290: 97 pgs: 97 active+clean; 267 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.7 MiB/s wr, 808 op/s 2024-09-20T22:19:26.793 INFO:teuthology.orchestra.run.smithi037.stdout: 5 47969 13.20 MB/sec warmup 104 sec latency 29.829 ms 2024-09-20T22:19:27.786 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:27 smithi037 ceph-mon[23718]: pgmap v291: 97 pgs: 97 active+clean; 271 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 7.1 MiB/s wr, 702 op/s 2024-09-20T22:19:27.793 INFO:teuthology.orchestra.run.smithi037.stdout: 5 48461 13.19 MB/sec warmup 105 sec latency 117.303 ms 2024-09-20T22:19:28.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:27 smithi194 ceph-mon[25957]: pgmap v291: 97 pgs: 97 active+clean; 271 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 7.1 MiB/s wr, 702 op/s 2024-09-20T22:19:28.794 INFO:teuthology.orchestra.run.smithi037.stdout: 5 48912 13.21 MB/sec warmup 106 sec latency 30.021 ms 2024-09-20T22:19:29.794 INFO:teuthology.orchestra.run.smithi037.stdout: 5 49514 13.33 MB/sec warmup 107 sec latency 45.479 ms 2024-09-20T22:19:29.841 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:29 smithi037 ceph-mon[23718]: pgmap v292: 97 pgs: 97 active+clean; 274 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 8.1 KiB/s rd, 9.0 MiB/s wr, 826 op/s 2024-09-20T22:19:30.052 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:29 smithi194 ceph-mon[25957]: pgmap v292: 97 pgs: 97 active+clean; 274 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 8.1 KiB/s rd, 9.0 MiB/s wr, 826 op/s 2024-09-20T22:19:30.794 INFO:teuthology.orchestra.run.smithi037.stdout: 5 50198 13.37 MB/sec warmup 108 sec latency 26.843 ms 2024-09-20T22:19:31.100 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:31 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:19:31.100 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:31 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:19:31.100 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:31 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:19:31.100 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:31 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:19:31.100 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:31 smithi037 ceph-mon[23718]: Upgrade: Need to upgrade myself (mgr.smithi037.lykblh) 2024-09-20T22:19:31.101 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:31 smithi037 ceph-mon[23718]: Failing over to other MGR 2024-09-20T22:19:31.101 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:31 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mgr fail", "who": "smithi037.lykblh"}]: dispatch 2024-09-20T22:19:31.101 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:31 smithi037 ceph-mon[23718]: osdmap e70: 8 total, 8 up, 8 in 2024-09-20T22:19:31.101 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:31 smithi037 ceph-mon[23718]: pgmap v294: 97 pgs: 97 active+clean; 274 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 9.5 KiB/s rd, 7.0 MiB/s wr, 657 op/s 2024-09-20T22:19:31.101 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:31 smithi037 ceph-mon[23718]: Standby manager daemon smithi037.lykblh started 2024-09-20T22:19:31.446 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:31 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:19:31.446 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:31 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:19:31.446 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:31 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' 2024-09-20T22:19:31.447 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:31 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:19:31.447 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:31 smithi194 ceph-mon[25957]: Upgrade: Need to upgrade myself (mgr.smithi037.lykblh) 2024-09-20T22:19:31.447 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:31 smithi194 ceph-mon[25957]: Failing over to other MGR 2024-09-20T22:19:31.447 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:31 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mgr fail", "who": "smithi037.lykblh"}]: dispatch 2024-09-20T22:19:31.447 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:31 smithi194 ceph-mon[25957]: osdmap e70: 8 total, 8 up, 8 in 2024-09-20T22:19:31.447 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:31 smithi194 ceph-mon[25957]: pgmap v294: 97 pgs: 97 active+clean; 274 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 9.5 KiB/s rd, 7.0 MiB/s wr, 657 op/s 2024-09-20T22:19:31.447 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:31 smithi194 ceph-mon[25957]: Standby manager daemon smithi037.lykblh started 2024-09-20T22:19:31.794 INFO:teuthology.orchestra.run.smithi037.stdout: 5 50733 13.33 MB/sec warmup 109 sec latency 32.771 ms 2024-09-20T22:19:32.359 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:32 smithi037 ceph-mon[23718]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "mgr fail", "who": "smithi037.lykblh"}]': finished 2024-09-20T22:19:32.359 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:32 smithi037 ceph-mon[23718]: mgrmap e23: smithi194.bkcglt(active, starting, since 0.998345s), standbys: smithi037.lykblh 2024-09-20T22:19:32.359 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:32 smithi037 ceph-mon[23718]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "mon metadata", "id": "smithi037"}]: dispatch 2024-09-20T22:19:32.359 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:32 smithi037 ceph-mon[23718]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "mon metadata", "id": "smithi194"}]: dispatch 2024-09-20T22:19:32.359 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:32 smithi037 ceph-mon[23718]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi037.sdtedi"}]: dispatch 2024-09-20T22:19:32.359 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:32 smithi037 ceph-mon[23718]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi194.lkroso"}]: dispatch 2024-09-20T22:19:32.359 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:32 smithi037 ceph-mon[23718]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "mgr metadata", "who": "smithi194.bkcglt", "id": "smithi194.bkcglt"}]: dispatch 2024-09-20T22:19:32.359 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:32 smithi037 ceph-mon[23718]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "mgr metadata", "who": "smithi037.lykblh", "id": "smithi037.lykblh"}]: dispatch 2024-09-20T22:19:32.360 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:32 smithi037 ceph-mon[23718]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-20T22:19:32.360 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:32 smithi037 ceph-mon[23718]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-20T22:19:32.360 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:32 smithi037 ceph-mon[23718]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-20T22:19:32.360 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:32 smithi037 ceph-mon[23718]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-20T22:19:32.360 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:32 smithi037 ceph-mon[23718]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-20T22:19:32.360 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:32 smithi037 ceph-mon[23718]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-20T22:19:32.360 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:32 smithi037 ceph-mon[23718]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-20T22:19:32.360 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:32 smithi037 ceph-mon[23718]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-20T22:19:32.360 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:32 smithi037 ceph-mon[23718]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-20T22:19:32.360 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:32 smithi037 ceph-mon[23718]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-20T22:19:32.360 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:32 smithi037 ceph-mon[23718]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-20T22:19:32.360 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:32 smithi037 ceph-mon[23718]: Manager daemon smithi194.bkcglt is now available 2024-09-20T22:19:32.360 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:32 smithi037 ceph-mon[23718]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:19:32.360 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:32 smithi037 ceph-mon[23718]: Checking for cert/key for grafana.smithi037 2024-09-20T22:19:32.361 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:32 smithi037 ceph-mon[23718]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:19:32.361 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:32 smithi037 ceph-mon[23718]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:19:32.361 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:32 smithi037 ceph-mon[23718]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:19:32.361 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:32 smithi037 ceph-mon[23718]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi194.bkcglt/mirror_snapshot_schedule"}]: dispatch 2024-09-20T22:19:32.361 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:32 smithi037 ceph-mon[23718]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi194.bkcglt/mirror_snapshot_schedule"}]: dispatch 2024-09-20T22:19:32.361 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:32 smithi037 ceph-mon[23718]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi194.bkcglt/trash_purge_schedule"}]: dispatch 2024-09-20T22:19:32.361 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:32 smithi037 ceph-mon[23718]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi194.bkcglt/trash_purge_schedule"}]: dispatch 2024-09-20T22:19:32.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:32 smithi194 ceph-mon[25957]: from='mgr.14204 172.21.15.37:0/2759463404' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "mgr fail", "who": "smithi037.lykblh"}]': finished 2024-09-20T22:19:32.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:32 smithi194 ceph-mon[25957]: mgrmap e23: smithi194.bkcglt(active, starting, since 0.998345s), standbys: smithi037.lykblh 2024-09-20T22:19:32.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:32 smithi194 ceph-mon[25957]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "mon metadata", "id": "smithi037"}]: dispatch 2024-09-20T22:19:32.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:32 smithi194 ceph-mon[25957]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "mon metadata", "id": "smithi194"}]: dispatch 2024-09-20T22:19:32.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:32 smithi194 ceph-mon[25957]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi037.sdtedi"}]: dispatch 2024-09-20T22:19:32.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:32 smithi194 ceph-mon[25957]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi194.lkroso"}]: dispatch 2024-09-20T22:19:32.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:32 smithi194 ceph-mon[25957]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "mgr metadata", "who": "smithi194.bkcglt", "id": "smithi194.bkcglt"}]: dispatch 2024-09-20T22:19:32.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:32 smithi194 ceph-mon[25957]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "mgr metadata", "who": "smithi037.lykblh", "id": "smithi037.lykblh"}]: dispatch 2024-09-20T22:19:32.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:32 smithi194 ceph-mon[25957]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-20T22:19:32.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:32 smithi194 ceph-mon[25957]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-20T22:19:32.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:32 smithi194 ceph-mon[25957]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-20T22:19:32.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:32 smithi194 ceph-mon[25957]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-20T22:19:32.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:32 smithi194 ceph-mon[25957]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-20T22:19:32.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:32 smithi194 ceph-mon[25957]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-20T22:19:32.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:32 smithi194 ceph-mon[25957]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-20T22:19:32.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:32 smithi194 ceph-mon[25957]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-20T22:19:32.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:32 smithi194 ceph-mon[25957]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-20T22:19:32.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:32 smithi194 ceph-mon[25957]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-20T22:19:32.596 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:32 smithi194 ceph-mon[25957]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-20T22:19:32.596 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:32 smithi194 ceph-mon[25957]: Manager daemon smithi194.bkcglt is now available 2024-09-20T22:19:32.596 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:32 smithi194 ceph-mon[25957]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:19:32.596 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:32 smithi194 ceph-mon[25957]: Checking for cert/key for grafana.smithi037 2024-09-20T22:19:32.596 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:32 smithi194 ceph-mon[25957]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:19:32.596 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:32 smithi194 ceph-mon[25957]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:19:32.596 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:32 smithi194 ceph-mon[25957]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:19:32.596 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:32 smithi194 ceph-mon[25957]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi194.bkcglt/mirror_snapshot_schedule"}]: dispatch 2024-09-20T22:19:32.596 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:32 smithi194 ceph-mon[25957]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi194.bkcglt/mirror_snapshot_schedule"}]: dispatch 2024-09-20T22:19:32.596 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:32 smithi194 ceph-mon[25957]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi194.bkcglt/trash_purge_schedule"}]: dispatch 2024-09-20T22:19:32.596 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:32 smithi194 ceph-mon[25957]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi194.bkcglt/trash_purge_schedule"}]: dispatch 2024-09-20T22:19:32.794 INFO:teuthology.orchestra.run.smithi037.stdout: 5 51327 13.31 MB/sec warmup 110 sec latency 40.257 ms 2024-09-20T22:19:33.369 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:33 smithi037 ceph-mon[23718]: Deploying cephadm binary to smithi194 2024-09-20T22:19:33.369 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:33 smithi037 ceph-mon[23718]: mgrmap e24: smithi194.bkcglt(active, since 2s), standbys: smithi037.lykblh 2024-09-20T22:19:33.369 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:33 smithi037 ceph-mon[23718]: pgmap v3: 97 pgs: 97 active+clean; 295 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-20T22:19:33.370 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:33 smithi037 ceph-mon[23718]: Deploying cephadm binary to smithi037 2024-09-20T22:19:33.370 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:33 smithi037 ceph-mon[23718]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:19:33.370 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:33 smithi037 ceph-mon[23718]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:19:33.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:33 smithi194 ceph-mon[25957]: Deploying cephadm binary to smithi194 2024-09-20T22:19:33.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:33 smithi194 ceph-mon[25957]: mgrmap e24: smithi194.bkcglt(active, since 2s), standbys: smithi037.lykblh 2024-09-20T22:19:33.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:33 smithi194 ceph-mon[25957]: pgmap v3: 97 pgs: 97 active+clean; 295 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-20T22:19:33.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:33 smithi194 ceph-mon[25957]: Deploying cephadm binary to smithi037 2024-09-20T22:19:33.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:33 smithi194 ceph-mon[25957]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:19:33.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:33 smithi194 ceph-mon[25957]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:19:33.794 INFO:teuthology.orchestra.run.smithi037.stdout: 5 51957 13.33 MB/sec warmup 111 sec latency 32.871 ms 2024-09-20T22:19:34.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:34 smithi194 ceph-mon[25957]: pgmap v4: 97 pgs: 97 active+clean; 295 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-20T22:19:34.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:34 smithi194 ceph-mon[25957]: [20/Sep/2024:22:19:33] ENGINE Bus STARTING 2024-09-20T22:19:34.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:34 smithi194 ceph-mon[25957]: [20/Sep/2024:22:19:33] ENGINE Serving on https://172.21.15.194:7150 2024-09-20T22:19:34.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:34 smithi194 ceph-mon[25957]: [20/Sep/2024:22:19:33] ENGINE Error in HTTPServer.serve 2024-09-20T22:19:34.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: Traceback (most recent call last): 2024-09-20T22:19:34.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-20T22:19:34.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: self._connections.run(self.expiration_interval) 2024-09-20T22:19:34.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-20T22:19:34.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: self._run(expiration_interval) 2024-09-20T22:19:34.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-20T22:19:34.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-20T22:19:34.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-20T22:19:34.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-20T22:19:34.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-20T22:19:34.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: s = self.context.wrap_socket( 2024-09-20T22:19:34.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-20T22:19:34.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: return self.sslsocket_class._create( 2024-09-20T22:19:34.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-20T22:19:34.346 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: self.do_handshake() 2024-09-20T22:19:34.346 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-20T22:19:34.346 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: self._sslobj.do_handshake() 2024-09-20T22:19:34.346 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-20T22:19:34.346 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:34 smithi194 ceph-mon[25957]: [20/Sep/2024:22:19:33] ENGINE Serving on http://172.21.15.194:8765 2024-09-20T22:19:34.346 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:34 smithi194 ceph-mon[25957]: [20/Sep/2024:22:19:33] ENGINE Bus STARTED 2024-09-20T22:19:34.373 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:34 smithi037 ceph-mon[23718]: pgmap v4: 97 pgs: 97 active+clean; 295 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-20T22:19:34.373 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:34 smithi037 ceph-mon[23718]: [20/Sep/2024:22:19:33] ENGINE Bus STARTING 2024-09-20T22:19:34.373 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:34 smithi037 ceph-mon[23718]: [20/Sep/2024:22:19:33] ENGINE Serving on https://172.21.15.194:7150 2024-09-20T22:19:34.373 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:34 smithi037 ceph-mon[23718]: [20/Sep/2024:22:19:33] ENGINE Error in HTTPServer.serve 2024-09-20T22:19:34.373 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: Traceback (most recent call last): 2024-09-20T22:19:34.373 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-20T22:19:34.374 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: self._connections.run(self.expiration_interval) 2024-09-20T22:19:34.374 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-20T22:19:34.374 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: self._run(expiration_interval) 2024-09-20T22:19:34.374 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-20T22:19:34.374 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-20T22:19:34.374 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-20T22:19:34.374 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-20T22:19:34.374 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-20T22:19:34.374 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: s = self.context.wrap_socket( 2024-09-20T22:19:34.374 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-20T22:19:34.374 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: return self.sslsocket_class._create( 2024-09-20T22:19:34.374 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-20T22:19:34.374 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: self.do_handshake() 2024-09-20T22:19:34.374 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-20T22:19:34.374 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: self._sslobj.do_handshake() 2024-09-20T22:19:34.375 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-20T22:19:34.375 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:34 smithi037 ceph-mon[23718]: [20/Sep/2024:22:19:33] ENGINE Serving on http://172.21.15.194:8765 2024-09-20T22:19:34.375 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:34 smithi037 ceph-mon[23718]: [20/Sep/2024:22:19:33] ENGINE Bus STARTED 2024-09-20T22:19:34.794 INFO:teuthology.orchestra.run.smithi037.stdout: 5 52590 13.39 MB/sec warmup 112 sec latency 29.681 ms 2024-09-20T22:19:35.379 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:35 smithi037 ceph-mon[23718]: mgrmap e25: smithi194.bkcglt(active, since 4s), standbys: smithi037.lykblh 2024-09-20T22:19:35.480 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:35 smithi194 ceph-mon[25957]: mgrmap e25: smithi194.bkcglt(active, since 4s), standbys: smithi037.lykblh 2024-09-20T22:19:35.794 INFO:teuthology.orchestra.run.smithi037.stdout: 5 53083 13.46 MB/sec warmup 113 sec latency 51.110 ms 2024-09-20T22:19:36.651 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:36 smithi037 ceph-mon[23718]: pgmap v5: 97 pgs: 97 active+clean; 295 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-20T22:19:36.651 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:36 smithi037 ceph-mon[23718]: mgrmap e26: smithi194.bkcglt(active, since 6s), standbys: smithi037.lykblh 2024-09-20T22:19:36.794 INFO:teuthology.orchestra.run.smithi037.stdout: 5 53376 13.41 MB/sec warmup 114 sec latency 79.354 ms 2024-09-20T22:19:36.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:36 smithi194 ceph-mon[25957]: pgmap v5: 97 pgs: 97 active+clean; 295 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-20T22:19:36.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:36 smithi194 ceph-mon[25957]: mgrmap e26: smithi194.bkcglt(active, since 6s), standbys: smithi037.lykblh 2024-09-20T22:19:37.794 INFO:teuthology.orchestra.run.smithi037.stdout: 5 53826 13.39 MB/sec warmup 115 sec latency 106.120 ms 2024-09-20T22:19:38.178 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:38 smithi037 ceph-mon[23718]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:19:38.178 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:38 smithi037 ceph-mon[23718]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:19:38.178 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:38 smithi037 ceph-mon[23718]: pgmap v6: 97 pgs: 97 active+clean; 295 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-20T22:19:38.178 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:38 smithi037 ceph-mon[23718]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:19:38.178 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:38 smithi037 ceph-mon[23718]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:19:38.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:38 smithi194 ceph-mon[25957]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:19:38.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:38 smithi194 ceph-mon[25957]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:19:38.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:38 smithi194 ceph-mon[25957]: pgmap v6: 97 pgs: 97 active+clean; 295 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-20T22:19:38.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:38 smithi194 ceph-mon[25957]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:19:38.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:38 smithi194 ceph-mon[25957]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:19:38.794 INFO:teuthology.orchestra.run.smithi037.stdout: 5 54249 13.35 MB/sec warmup 116 sec latency 35.491 ms 2024-09-20T22:19:39.794 INFO:teuthology.orchestra.run.smithi037.stdout: 5 54858 13.33 MB/sec warmup 117 sec latency 44.637 ms 2024-09-20T22:19:40.702 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:40 smithi037 ceph-mon[23718]: pgmap v7: 97 pgs: 97 active+clean; 301 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 39 KiB/s rd, 5.6 MiB/s wr, 518 op/s 2024-09-20T22:19:40.794 INFO:teuthology.orchestra.run.smithi037.stdout: 5 55507 13.35 MB/sec warmup 118 sec latency 30.742 ms 2024-09-20T22:19:40.795 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:40 smithi194 ceph-mon[25957]: pgmap v7: 97 pgs: 97 active+clean; 301 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 39 KiB/s rd, 5.6 MiB/s wr, 518 op/s 2024-09-20T22:19:41.717 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:41 smithi037 ceph-mon[23718]: from='mgr.? 172.21.15.37:0/2342042235' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi037.lykblh/crt"}]: dispatch 2024-09-20T22:19:41.717 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:41 smithi037 ceph-mon[23718]: Standby manager daemon smithi037.lykblh restarted 2024-09-20T22:19:41.717 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:41 smithi037 ceph-mon[23718]: Standby manager daemon smithi037.lykblh started 2024-09-20T22:19:41.717 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:41 smithi037 ceph-mon[23718]: from='mgr.? 172.21.15.37:0/2342042235' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-20T22:19:41.718 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:41 smithi037 ceph-mon[23718]: from='mgr.? 172.21.15.37:0/2342042235' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi037.lykblh/key"}]: dispatch 2024-09-20T22:19:41.718 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:41 smithi037 ceph-mon[23718]: from='mgr.? 172.21.15.37:0/2342042235' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-20T22:19:41.785 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:41 smithi194 ceph-mon[25957]: from='mgr.? 172.21.15.37:0/2342042235' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi037.lykblh/crt"}]: dispatch 2024-09-20T22:19:41.785 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:41 smithi194 ceph-mon[25957]: Standby manager daemon smithi037.lykblh restarted 2024-09-20T22:19:41.785 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:41 smithi194 ceph-mon[25957]: Standby manager daemon smithi037.lykblh started 2024-09-20T22:19:41.785 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:41 smithi194 ceph-mon[25957]: from='mgr.? 172.21.15.37:0/2342042235' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-20T22:19:41.786 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:41 smithi194 ceph-mon[25957]: from='mgr.? 172.21.15.37:0/2342042235' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi037.lykblh/key"}]: dispatch 2024-09-20T22:19:41.786 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:41 smithi194 ceph-mon[25957]: from='mgr.? 172.21.15.37:0/2342042235' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-20T22:19:41.794 INFO:teuthology.orchestra.run.smithi037.stdout: 5 56114 13.41 MB/sec warmup 119 sec latency 35.763 ms 2024-09-20T22:19:42.760 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:42 smithi037 ceph-mon[23718]: pgmap v8: 97 pgs: 97 active+clean; 301 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 30 KiB/s rd, 4.4 MiB/s wr, 403 op/s 2024-09-20T22:19:42.761 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:42 smithi037 ceph-mon[23718]: mgrmap e27: smithi194.bkcglt(active, since 11s), standbys: smithi037.lykblh 2024-09-20T22:19:42.761 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:42 smithi037 ceph-mon[23718]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:19:42.761 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:42 smithi037 ceph-mon[23718]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:19:42.761 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:42 smithi037 ceph-mon[23718]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-20T22:19:42.761 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:42 smithi037 ceph-mon[23718]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-20T22:19:42.761 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:42 smithi037 ceph-mon[23718]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-20T22:19:42.761 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:42 smithi037 ceph-mon[23718]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-20T22:19:42.761 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:42 smithi037 ceph-mon[23718]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-20T22:19:42.761 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:42 smithi037 ceph-mon[23718]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-20T22:19:42.761 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:42 smithi037 ceph-mon[23718]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-20T22:19:42.761 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:42 smithi037 ceph-mon[23718]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-20T22:19:42.762 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:42 smithi037 ceph-mon[23718]: Adjusting osd_memory_target on smithi194 to 2704M 2024-09-20T22:19:42.762 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:42 smithi037 ceph-mon[23718]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:19:42.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:42 smithi194 ceph-mon[25957]: pgmap v8: 97 pgs: 97 active+clean; 301 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 30 KiB/s rd, 4.4 MiB/s wr, 403 op/s 2024-09-20T22:19:42.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:42 smithi194 ceph-mon[25957]: mgrmap e27: smithi194.bkcglt(active, since 11s), standbys: smithi037.lykblh 2024-09-20T22:19:42.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:42 smithi194 ceph-mon[25957]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:19:42.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:42 smithi194 ceph-mon[25957]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:19:42.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:42 smithi194 ceph-mon[25957]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-20T22:19:42.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:42 smithi194 ceph-mon[25957]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-20T22:19:42.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:42 smithi194 ceph-mon[25957]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-20T22:19:42.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:42 smithi194 ceph-mon[25957]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-20T22:19:42.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:42 smithi194 ceph-mon[25957]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-20T22:19:42.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:42 smithi194 ceph-mon[25957]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-20T22:19:42.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:42 smithi194 ceph-mon[25957]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-20T22:19:42.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:42 smithi194 ceph-mon[25957]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-20T22:19:42.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:42 smithi194 ceph-mon[25957]: Adjusting osd_memory_target on smithi194 to 2704M 2024-09-20T22:19:42.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:42 smithi194 ceph-mon[25957]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:19:43.794 INFO:teuthology.orchestra.run.smithi037.stdout: 5 56920 12.69 MB/sec execute 1 sec latency 65.193 ms 2024-09-20T22:19:44.727 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:44 smithi037 ceph-mon[23718]: pgmap v9: 97 pgs: 97 active+clean; 305 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 30 KiB/s rd, 6.9 MiB/s wr, 679 op/s 2024-09-20T22:19:44.727 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:44 smithi037 ceph-mon[23718]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:19:44.728 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:44 smithi037 ceph-mon[23718]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:19:44.794 INFO:teuthology.orchestra.run.smithi037.stdout: 5 57302 11.46 MB/sec execute 2 sec latency 27.672 ms 2024-09-20T22:19:44.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:44 smithi194 ceph-mon[25957]: pgmap v9: 97 pgs: 97 active+clean; 305 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 30 KiB/s rd, 6.9 MiB/s wr, 679 op/s 2024-09-20T22:19:44.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:44 smithi194 ceph-mon[25957]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:19:44.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:44 smithi194 ceph-mon[25957]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:19:45.799 INFO:teuthology.orchestra.run.smithi037.stdout: 5 57829 10.73 MB/sec execute 3 sec latency 37.087 ms 2024-09-20T22:19:46.301 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:46 smithi037 ceph-mon[23718]: pgmap v10: 97 pgs: 97 active+clean; 305 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 27 KiB/s rd, 6.3 MiB/s wr, 622 op/s 2024-09-20T22:19:46.302 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:46 smithi037 ceph-mon[23718]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:19:46.302 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:46 smithi037 ceph-mon[23718]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:19:46.302 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:46 smithi037 ceph-mon[23718]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-20T22:19:46.302 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:46 smithi037 ceph-mon[23718]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-20T22:19:46.302 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:46 smithi037 ceph-mon[23718]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-20T22:19:46.302 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:46 smithi037 ceph-mon[23718]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-20T22:19:46.302 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:46 smithi037 ceph-mon[23718]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-20T22:19:46.302 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:46 smithi037 ceph-mon[23718]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-20T22:19:46.302 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:46 smithi037 ceph-mon[23718]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-20T22:19:46.302 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:46 smithi037 ceph-mon[23718]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-20T22:19:46.302 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:46 smithi037 ceph-mon[23718]: Adjusting osd_memory_target on smithi037 to 1937M 2024-09-20T22:19:46.303 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:46 smithi037 ceph-mon[23718]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:19:46.303 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:46 smithi037 ceph-mon[23718]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:19:46.303 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:46 smithi037 ceph-mon[23718]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:19:46.303 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:46 smithi037 ceph-mon[23718]: Updating smithi037:/etc/ceph/ceph.conf 2024-09-20T22:19:46.303 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:46 smithi037 ceph-mon[23718]: Updating smithi194:/etc/ceph/ceph.conf 2024-09-20T22:19:46.303 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:46 smithi037 ceph-mon[23718]: Updating smithi194:/var/lib/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/config/ceph.conf 2024-09-20T22:19:46.303 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:46 smithi037 ceph-mon[23718]: Updating smithi037:/var/lib/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/config/ceph.conf 2024-09-20T22:19:46.303 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:46 smithi037 ceph-mon[23718]: Updating smithi194:/etc/ceph/ceph.client.admin.keyring 2024-09-20T22:19:46.303 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:46 smithi037 ceph-mon[23718]: Updating smithi037:/etc/ceph/ceph.client.admin.keyring 2024-09-20T22:19:46.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:46 smithi194 ceph-mon[25957]: pgmap v10: 97 pgs: 97 active+clean; 305 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 27 KiB/s rd, 6.3 MiB/s wr, 622 op/s 2024-09-20T22:19:46.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:46 smithi194 ceph-mon[25957]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:19:46.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:46 smithi194 ceph-mon[25957]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:19:46.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:46 smithi194 ceph-mon[25957]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-20T22:19:46.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:46 smithi194 ceph-mon[25957]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-20T22:19:46.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:46 smithi194 ceph-mon[25957]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-20T22:19:46.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:46 smithi194 ceph-mon[25957]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-20T22:19:46.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:46 smithi194 ceph-mon[25957]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-20T22:19:46.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:46 smithi194 ceph-mon[25957]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-20T22:19:46.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:46 smithi194 ceph-mon[25957]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-20T22:19:46.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:46 smithi194 ceph-mon[25957]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-20T22:19:46.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:46 smithi194 ceph-mon[25957]: Adjusting osd_memory_target on smithi037 to 1937M 2024-09-20T22:19:46.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:46 smithi194 ceph-mon[25957]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:19:46.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:46 smithi194 ceph-mon[25957]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:19:46.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:46 smithi194 ceph-mon[25957]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:19:46.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:46 smithi194 ceph-mon[25957]: Updating smithi037:/etc/ceph/ceph.conf 2024-09-20T22:19:46.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:46 smithi194 ceph-mon[25957]: Updating smithi194:/etc/ceph/ceph.conf 2024-09-20T22:19:46.596 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:46 smithi194 ceph-mon[25957]: Updating smithi194:/var/lib/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/config/ceph.conf 2024-09-20T22:19:46.596 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:46 smithi194 ceph-mon[25957]: Updating smithi037:/var/lib/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/config/ceph.conf 2024-09-20T22:19:46.596 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:46 smithi194 ceph-mon[25957]: Updating smithi194:/etc/ceph/ceph.client.admin.keyring 2024-09-20T22:19:46.596 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:46 smithi194 ceph-mon[25957]: Updating smithi037:/etc/ceph/ceph.client.admin.keyring 2024-09-20T22:19:46.799 INFO:teuthology.orchestra.run.smithi037.stdout: 5 58443 11.08 MB/sec execute 4 sec latency 32.392 ms 2024-09-20T22:19:46.961 INFO:teuthology.orchestra.run.smithi037.stdout:true 2024-09-20T22:19:47.558 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:47 smithi037 ceph-mon[23718]: Updating smithi194:/var/lib/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/config/ceph.client.admin.keyring 2024-09-20T22:19:47.559 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:47 smithi037 ceph-mon[23718]: Updating smithi037:/var/lib/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/config/ceph.client.admin.keyring 2024-09-20T22:19:47.559 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:47 smithi037 ceph-mon[23718]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:19:47.559 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:47 smithi037 ceph-mon[23718]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:19:47.559 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:47 smithi037 ceph-mon[23718]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:19:47.559 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:47 smithi037 ceph-mon[23718]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:19:47.559 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:47 smithi037 ceph-mon[23718]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:19:47.559 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:47 smithi037 ceph-mon[23718]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:19:47.559 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:47 smithi037 ceph-mon[23718]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:19:47.559 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:47 smithi037 ceph-mon[23718]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:19:47.559 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:47 smithi037 ceph-mon[23718]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:19:47.559 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:47 smithi037 ceph-mon[23718]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:19:47.559 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:47 smithi037 ceph-mon[23718]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:19:47.559 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:47 smithi037 ceph-mon[23718]: Reconfiguring alertmanager.smithi037 (dependencies changed)... 2024-09-20T22:19:47.559 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:47 smithi037 ceph-mon[23718]: Deploying daemon alertmanager.smithi037 on smithi037 2024-09-20T22:19:47.560 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:47 smithi037 ceph-mon[23718]: from='client.24493 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:19:47.799 INFO:teuthology.orchestra.run.smithi037.stdout: 5 59042 11.87 MB/sec execute 5 sec latency 25.912 ms 2024-09-20T22:19:47.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:47 smithi194 ceph-mon[25957]: Updating smithi194:/var/lib/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/config/ceph.client.admin.keyring 2024-09-20T22:19:47.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:47 smithi194 ceph-mon[25957]: Updating smithi037:/var/lib/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/config/ceph.client.admin.keyring 2024-09-20T22:19:47.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:47 smithi194 ceph-mon[25957]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:19:47.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:47 smithi194 ceph-mon[25957]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:19:47.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:47 smithi194 ceph-mon[25957]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:19:47.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:47 smithi194 ceph-mon[25957]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:19:47.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:47 smithi194 ceph-mon[25957]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:19:47.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:47 smithi194 ceph-mon[25957]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:19:47.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:47 smithi194 ceph-mon[25957]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:19:47.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:47 smithi194 ceph-mon[25957]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:19:47.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:47 smithi194 ceph-mon[25957]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:19:47.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:47 smithi194 ceph-mon[25957]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:19:47.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:47 smithi194 ceph-mon[25957]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:19:47.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:47 smithi194 ceph-mon[25957]: Reconfiguring alertmanager.smithi037 (dependencies changed)... 2024-09-20T22:19:47.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:47 smithi194 ceph-mon[25957]: Deploying daemon alertmanager.smithi037 on smithi037 2024-09-20T22:19:47.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:47 smithi194 ceph-mon[25957]: from='client.24493 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:19:48.139 INFO:teuthology.orchestra.run.smithi037.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T22:19:48.140 INFO:teuthology.orchestra.run.smithi037.stdout:alertmanager.smithi037 smithi037 *:9093,9094 running (7m) 3s ago 8m 25.8M - 0.23.0 ba2b418f427c bb407b78f0c9 2024-09-20T22:19:48.140 INFO:teuthology.orchestra.run.smithi037.stdout:crash.smithi037 smithi037 running (8m) 3s ago 8m 6400k - 17.2.0 e1d6a67b021e 88301022bdc8 2024-09-20T22:19:48.140 INFO:teuthology.orchestra.run.smithi037.stdout:crash.smithi194 smithi194 running (7m) 11s ago 7m 6405k - 17.2.0 e1d6a67b021e b553601d7ca5 2024-09-20T22:19:48.140 INFO:teuthology.orchestra.run.smithi037.stdout:grafana.smithi037 smithi037 *:3000 running (6m) 3s ago 8m 52.8M - dad864ee21e9 3ccb7bb6f7ae 2024-09-20T22:19:48.140 INFO:teuthology.orchestra.run.smithi037.stdout:mds.foofs.smithi037.sdtedi smithi037 running (2m) 3s ago 2m 34.8M - 17.2.0 e1d6a67b021e 0a7898d25b8d 2024-09-20T22:19:48.140 INFO:teuthology.orchestra.run.smithi037.stdout:mds.foofs.smithi194.lkroso smithi194 running (2m) 11s ago 2m 14.2M - 17.2.0 e1d6a67b021e 17736de30008 2024-09-20T22:19:48.140 INFO:teuthology.orchestra.run.smithi037.stdout:mgr.smithi037.lykblh smithi037 *:9283 running (9m) 3s ago 9m 139M - 17.2.0 e1d6a67b021e 3ef1f99a6ba9 2024-09-20T22:19:48.140 INFO:teuthology.orchestra.run.smithi037.stdout:mgr.smithi194.bkcglt smithi194 *:8443,9283 running (37s) 11s ago 7m 478M - 19.1.1-367-g9e2ac70b cc9251b744a3 e0af2512175a 2024-09-20T22:19:48.140 INFO:teuthology.orchestra.run.smithi037.stdout:mon.smithi037 smithi037 running (9m) 3s ago 10m 49.5M 2048M 17.2.0 e1d6a67b021e 71f591122786 2024-09-20T22:19:48.140 INFO:teuthology.orchestra.run.smithi037.stdout:mon.smithi194 smithi194 running (7m) 11s ago 7m 42.5M 2048M 17.2.0 e1d6a67b021e 0b8814ae6c66 2024-09-20T22:19:48.140 INFO:teuthology.orchestra.run.smithi037.stdout:nfs.foo.0.0.smithi037.spabdc smithi037 *:2049 running (2m) 3s ago 2m 253M - 4.0 e1d6a67b021e 858ff9d558a2 2024-09-20T22:19:48.140 INFO:teuthology.orchestra.run.smithi037.stdout:nfs.foo.1.0.smithi194.qzjyej smithi194 *:2049 running (2m) 11s ago 2m 77.7M - 4.0 e1d6a67b021e c07fe0eb7c75 2024-09-20T22:19:48.140 INFO:teuthology.orchestra.run.smithi037.stdout:node-exporter.smithi037 smithi037 *:9100 running (8m) 3s ago 8m 17.0M - 1.3.1 1dbe0e931976 1f92060f034e 2024-09-20T22:19:48.140 INFO:teuthology.orchestra.run.smithi037.stdout:node-exporter.smithi194 smithi194 *:9100 running (7m) 11s ago 7m 21.0M - 1.3.1 1dbe0e931976 3a80a2ef6678 2024-09-20T22:19:48.141 INFO:teuthology.orchestra.run.smithi037.stdout:osd.0 smithi037 running (7m) 3s ago 7m 256M 1937M 17.2.0 e1d6a67b021e e54e84fd5030 2024-09-20T22:19:48.141 INFO:teuthology.orchestra.run.smithi037.stdout:osd.1 smithi037 running (6m) 3s ago 6m 193M 1937M 17.2.0 e1d6a67b021e 09647c133032 2024-09-20T22:19:48.141 INFO:teuthology.orchestra.run.smithi037.stdout:osd.2 smithi037 running (6m) 3s ago 6m 158M 1937M 17.2.0 e1d6a67b021e 6c15b1175256 2024-09-20T22:19:48.141 INFO:teuthology.orchestra.run.smithi037.stdout:osd.3 smithi037 running (5m) 3s ago 5m 208M 1937M 17.2.0 e1d6a67b021e 3a22952c6475 2024-09-20T22:19:48.141 INFO:teuthology.orchestra.run.smithi037.stdout:osd.4 smithi194 running (5m) 11s ago 5m 224M 2704M 17.2.0 e1d6a67b021e fa107be8915c 2024-09-20T22:19:48.141 INFO:teuthology.orchestra.run.smithi037.stdout:osd.5 smithi194 running (4m) 11s ago 4m 216M 2704M 17.2.0 e1d6a67b021e 290b21cbd131 2024-09-20T22:19:48.141 INFO:teuthology.orchestra.run.smithi037.stdout:osd.6 smithi194 running (4m) 11s ago 4m 203M 2704M 17.2.0 e1d6a67b021e 27f0720639ed 2024-09-20T22:19:48.141 INFO:teuthology.orchestra.run.smithi037.stdout:osd.7 smithi194 running (4m) 11s ago 4m 307M 2704M 17.2.0 e1d6a67b021e 62d163f4fd92 2024-09-20T22:19:48.141 INFO:teuthology.orchestra.run.smithi037.stdout:prometheus.smithi037 smithi037 *:9095 running (7m) 3s ago 7m 64.0M - 2.33.4 514e6a882f6e 32020a1cc2f4 2024-09-20T22:19:48.804 INFO:teuthology.orchestra.run.smithi037.stdout: 5 59629 13.18 MB/sec execute 6 sec latency 35.893 ms 2024-09-20T22:19:48.813 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:48 smithi037 ceph-mon[23718]: pgmap v11: 97 pgs: 97 active+clean; 305 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 27 KiB/s rd, 6.3 MiB/s wr, 622 op/s 2024-09-20T22:19:48.813 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:48 smithi037 ceph-mon[23718]: from='client.14710 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:19:48.843 INFO:teuthology.orchestra.run.smithi037.stdout:{ 2024-09-20T22:19:48.843 INFO:teuthology.orchestra.run.smithi037.stdout: "mon": { 2024-09-20T22:19:48.843 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T22:19:48.843 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:19:48.843 INFO:teuthology.orchestra.run.smithi037.stdout: "mgr": { 2024-09-20T22:19:48.844 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-20T22:19:48.844 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 1 2024-09-20T22:19:48.844 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:19:48.844 INFO:teuthology.orchestra.run.smithi037.stdout: "osd": { 2024-09-20T22:19:48.844 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-20T22:19:48.844 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:19:48.844 INFO:teuthology.orchestra.run.smithi037.stdout: "mds": { 2024-09-20T22:19:48.844 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T22:19:48.844 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:19:48.844 INFO:teuthology.orchestra.run.smithi037.stdout: "overall": { 2024-09-20T22:19:48.844 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-09-20T22:19:48.844 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 1 2024-09-20T22:19:48.844 INFO:teuthology.orchestra.run.smithi037.stdout: } 2024-09-20T22:19:48.844 INFO:teuthology.orchestra.run.smithi037.stdout:} 2024-09-20T22:19:49.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:48 smithi194 ceph-mon[25957]: pgmap v11: 97 pgs: 97 active+clean; 305 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 27 KiB/s rd, 6.3 MiB/s wr, 622 op/s 2024-09-20T22:19:49.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:48 smithi194 ceph-mon[25957]: from='client.14710 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:19:49.351 INFO:teuthology.orchestra.run.smithi037.stdout:{ 2024-09-20T22:19:49.351 INFO:teuthology.orchestra.run.smithi037.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5", 2024-09-20T22:19:49.351 INFO:teuthology.orchestra.run.smithi037.stdout: "in_progress": true, 2024-09-20T22:19:49.351 INFO:teuthology.orchestra.run.smithi037.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-20T22:19:49.351 INFO:teuthology.orchestra.run.smithi037.stdout: "services_complete": [], 2024-09-20T22:19:49.351 INFO:teuthology.orchestra.run.smithi037.stdout: "progress": "1/23 daemons upgraded", 2024-09-20T22:19:49.351 INFO:teuthology.orchestra.run.smithi037.stdout: "message": "", 2024-09-20T22:19:49.351 INFO:teuthology.orchestra.run.smithi037.stdout: "is_paused": false 2024-09-20T22:19:49.351 INFO:teuthology.orchestra.run.smithi037.stdout:} 2024-09-20T22:19:49.804 INFO:teuthology.orchestra.run.smithi037.stdout: 5 60141 14.15 MB/sec execute 7 sec latency 35.806 ms 2024-09-20T22:19:49.838 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:49 smithi037 ceph-mon[23718]: from='client.14714 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:19:49.838 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:49 smithi037 ceph-mon[23718]: from='client.? 172.21.15.37:0/3099152553' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:19:49.977 INFO:teuthology.orchestra.run.smithi037.stdout:HEALTH_OK 2024-09-20T22:19:50.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:49 smithi194 ceph-mon[25957]: from='client.14714 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:19:50.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:49 smithi194 ceph-mon[25957]: from='client.? 172.21.15.37:0/3099152553' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:19:50.804 INFO:teuthology.orchestra.run.smithi037.stdout: 5 60474 13.52 MB/sec execute 8 sec latency 40.885 ms 2024-09-20T22:19:50.849 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:50 smithi037 ceph-mon[23718]: pgmap v12: 97 pgs: 97 active+clean; 314 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 31 KiB/s rd, 9.2 MiB/s wr, 911 op/s 2024-09-20T22:19:50.849 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:50 smithi037 ceph-mon[23718]: from='client.14720 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:19:50.849 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:50 smithi037 ceph-mon[23718]: from='client.? 172.21.15.37:0/499080770' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T22:19:51.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:50 smithi194 ceph-mon[25957]: pgmap v12: 97 pgs: 97 active+clean; 314 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 31 KiB/s rd, 9.2 MiB/s wr, 911 op/s 2024-09-20T22:19:51.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:50 smithi194 ceph-mon[25957]: from='client.14720 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:19:51.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:50 smithi194 ceph-mon[25957]: from='client.? 172.21.15.37:0/499080770' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T22:19:51.804 INFO:teuthology.orchestra.run.smithi037.stdout: 5 60846 13.13 MB/sec execute 9 sec latency 44.643 ms 2024-09-20T22:19:52.804 INFO:teuthology.orchestra.run.smithi037.stdout: 5 61268 12.63 MB/sec execute 10 sec latency 75.717 ms 2024-09-20T22:19:52.864 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:52 smithi037 ceph-mon[23718]: pgmap v13: 97 pgs: 97 active+clean; 314 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 9.0 KiB/s rd, 6.0 MiB/s wr, 610 op/s 2024-09-20T22:19:53.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:52 smithi194 ceph-mon[25957]: pgmap v13: 97 pgs: 97 active+clean; 314 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 9.0 KiB/s rd, 6.0 MiB/s wr, 610 op/s 2024-09-20T22:19:53.804 INFO:teuthology.orchestra.run.smithi037.stdout: 5 61778 12.35 MB/sec execute 11 sec latency 43.398 ms 2024-09-20T22:19:54.804 INFO:teuthology.orchestra.run.smithi037.stdout: 5 62281 12.38 MB/sec execute 12 sec latency 47.091 ms 2024-09-20T22:19:54.898 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:54 smithi037 ceph-mon[23718]: pgmap v14: 97 pgs: 97 active+clean; 329 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 9.2 MiB/s wr, 885 op/s 2024-09-20T22:19:55.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:54 smithi194 ceph-mon[25957]: pgmap v14: 97 pgs: 97 active+clean; 329 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 9.2 MiB/s wr, 885 op/s 2024-09-20T22:19:55.804 INFO:teuthology.orchestra.run.smithi037.stdout: 5 62802 12.37 MB/sec execute 13 sec latency 29.748 ms 2024-09-20T22:19:56.804 INFO:teuthology.orchestra.run.smithi037.stdout: 5 63268 12.56 MB/sec execute 14 sec latency 62.240 ms 2024-09-20T22:19:56.924 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:56 smithi037 ceph-mon[23718]: pgmap v15: 97 pgs: 97 active+clean; 329 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 6.2 MiB/s wr, 565 op/s 2024-09-20T22:19:57.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:56 smithi194 ceph-mon[25957]: pgmap v15: 97 pgs: 97 active+clean; 329 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 6.2 MiB/s wr, 565 op/s 2024-09-20T22:19:57.804 INFO:teuthology.orchestra.run.smithi037.stdout: 5 63724 13.12 MB/sec execute 15 sec latency 52.007 ms 2024-09-20T22:19:58.804 INFO:teuthology.orchestra.run.smithi037.stdout: 5 64081 12.94 MB/sec execute 16 sec latency 104.256 ms 2024-09-20T22:19:58.941 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:19:58 smithi037 ceph-mon[23718]: pgmap v16: 97 pgs: 97 active+clean; 329 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 6.2 MiB/s wr, 565 op/s 2024-09-20T22:19:59.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:19:58 smithi194 ceph-mon[25957]: pgmap v16: 97 pgs: 97 active+clean; 329 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 6.2 MiB/s wr, 565 op/s 2024-09-20T22:19:59.804 INFO:teuthology.orchestra.run.smithi037.stdout: 5 64505 12.64 MB/sec execute 17 sec latency 37.317 ms 2024-09-20T22:20:00.805 INFO:teuthology.orchestra.run.smithi037.stdout: 5 64922 12.41 MB/sec execute 18 sec latency 126.086 ms 2024-09-20T22:20:00.966 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:00 smithi037 ceph-mon[23718]: pgmap v17: 97 pgs: 97 active+clean; 332 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 9.0 MiB/s wr, 855 op/s 2024-09-20T22:20:00.966 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:00 smithi037 ceph-mon[23718]: overall HEALTH_OK 2024-09-20T22:20:01.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:00 smithi194 ceph-mon[25957]: pgmap v17: 97 pgs: 97 active+clean; 332 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 9.0 MiB/s wr, 855 op/s 2024-09-20T22:20:01.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:00 smithi194 ceph-mon[25957]: overall HEALTH_OK 2024-09-20T22:20:01.805 INFO:teuthology.orchestra.run.smithi037.stdout: 5 65453 12.32 MB/sec execute 19 sec latency 31.638 ms 2024-09-20T22:20:01.973 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:01 smithi037 ceph-mon[23718]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:20:02.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:01 smithi194 ceph-mon[25957]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:20:02.805 INFO:teuthology.orchestra.run.smithi037.stdout: 5 66156 12.54 MB/sec execute 20 sec latency 25.865 ms 2024-09-20T22:20:02.861 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:02 smithi037 ceph-mon[23718]: pgmap v18: 97 pgs: 97 active+clean; 332 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 6.1 MiB/s wr, 565 op/s 2024-09-20T22:20:03.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:02 smithi194 ceph-mon[25957]: pgmap v18: 97 pgs: 97 active+clean; 332 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 6.1 MiB/s wr, 565 op/s 2024-09-20T22:20:03.805 INFO:teuthology.orchestra.run.smithi037.stdout: 5 66712 12.73 MB/sec execute 21 sec latency 71.689 ms 2024-09-20T22:20:04.543 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:04 smithi037 ceph-mon[23718]: pgmap v19: 97 pgs: 97 active+clean; 339 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.8 MiB/s wr, 913 op/s 2024-09-20T22:20:04.543 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:04 smithi037 ceph-mon[23718]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:20:04.543 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:04 smithi037 ceph-mon[23718]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:20:04.543 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:04 smithi037 ceph-mon[23718]: Reconfiguring grafana.smithi037 (dependencies changed)... 2024-09-20T22:20:04.544 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:04 smithi037 ceph-mon[23718]: Regenerating cephadm self-signed grafana TLS certificates 2024-09-20T22:20:04.544 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:04 smithi037 ceph-mon[23718]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:20:04.544 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:04 smithi037 ceph-mon[23718]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:20:04.544 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:04 smithi037 ceph-mon[23718]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-20T22:20:04.544 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:04 smithi037 ceph-mon[23718]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-20T22:20:04.544 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:04 smithi037 ceph-mon[23718]: Reconfiguring daemon grafana.smithi037 on smithi037 2024-09-20T22:20:04.805 INFO:teuthology.orchestra.run.smithi037.stdout: 5 67190 13.06 MB/sec execute 22 sec latency 65.692 ms 2024-09-20T22:20:04.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:04 smithi194 ceph-mon[25957]: pgmap v19: 97 pgs: 97 active+clean; 339 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.8 MiB/s wr, 913 op/s 2024-09-20T22:20:04.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:04 smithi194 ceph-mon[25957]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:20:04.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:04 smithi194 ceph-mon[25957]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:20:04.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:04 smithi194 ceph-mon[25957]: Reconfiguring grafana.smithi037 (dependencies changed)... 2024-09-20T22:20:04.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:04 smithi194 ceph-mon[25957]: Regenerating cephadm self-signed grafana TLS certificates 2024-09-20T22:20:04.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:04 smithi194 ceph-mon[25957]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:20:04.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:04 smithi194 ceph-mon[25957]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:20:04.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:04 smithi194 ceph-mon[25957]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-20T22:20:04.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:04 smithi194 ceph-mon[25957]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-20T22:20:04.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:04 smithi194 ceph-mon[25957]: Reconfiguring daemon grafana.smithi037 on smithi037 2024-09-20T22:20:05.805 INFO:teuthology.orchestra.run.smithi037.stdout: 5 67602 13.03 MB/sec execute 23 sec latency 51.297 ms 2024-09-20T22:20:06.576 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:06 smithi037 ceph-mon[23718]: pgmap v20: 97 pgs: 97 active+clean; 339 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.5 MiB/s wr, 638 op/s 2024-09-20T22:20:06.805 INFO:teuthology.orchestra.run.smithi037.stdout: 5 68157 13.03 MB/sec execute 24 sec latency 20.910 ms 2024-09-20T22:20:06.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:06 smithi194 ceph-mon[25957]: pgmap v20: 97 pgs: 97 active+clean; 339 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.5 MiB/s wr, 638 op/s 2024-09-20T22:20:07.806 INFO:teuthology.orchestra.run.smithi037.stdout: 5 68751 13.05 MB/sec execute 25 sec latency 32.036 ms 2024-09-20T22:20:08.606 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:08 smithi037 ceph-mon[23718]: pgmap v21: 97 pgs: 97 active+clean; 339 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.5 MiB/s wr, 638 op/s 2024-09-20T22:20:08.805 INFO:teuthology.orchestra.run.smithi037.stdout: 5 69429 13.08 MB/sec execute 26 sec latency 55.792 ms 2024-09-20T22:20:08.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:08 smithi194 ceph-mon[25957]: pgmap v21: 97 pgs: 97 active+clean; 339 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.5 MiB/s wr, 638 op/s 2024-09-20T22:20:09.805 INFO:teuthology.orchestra.run.smithi037.stdout: 5 70132 13.34 MB/sec execute 27 sec latency 30.265 ms 2024-09-20T22:20:10.634 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:10 smithi037 ceph-mon[23718]: pgmap v22: 97 pgs: 97 active+clean; 348 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 9.1 MiB/s wr, 952 op/s 2024-09-20T22:20:10.806 INFO:teuthology.orchestra.run.smithi037.stdout: 5 70768 13.78 MB/sec execute 28 sec latency 29.203 ms 2024-09-20T22:20:10.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:10 smithi194 ceph-mon[25957]: pgmap v22: 97 pgs: 97 active+clean; 348 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 9.1 MiB/s wr, 952 op/s 2024-09-20T22:20:11.806 INFO:teuthology.orchestra.run.smithi037.stdout: 5 71202 13.74 MB/sec execute 29 sec latency 30.575 ms 2024-09-20T22:20:12.649 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:12 smithi037 ceph-mon[23718]: pgmap v23: 97 pgs: 97 active+clean; 348 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 6.2 MiB/s wr, 662 op/s 2024-09-20T22:20:12.649 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:12 smithi037 ceph-mon[23718]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:20:12.649 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:12 smithi037 ceph-mon[23718]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:20:12.650 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:12 smithi037 ceph-mon[23718]: Reconfiguring node-exporter.smithi037 (dependencies changed)... 2024-09-20T22:20:12.650 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:12 smithi037 ceph-mon[23718]: Deploying daemon node-exporter.smithi037 on smithi037 2024-09-20T22:20:12.806 INFO:teuthology.orchestra.run.smithi037.stdout: 5 71693 13.61 MB/sec execute 30 sec latency 35.773 ms 2024-09-20T22:20:12.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:12 smithi194 ceph-mon[25957]: pgmap v23: 97 pgs: 97 active+clean; 348 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 6.2 MiB/s wr, 662 op/s 2024-09-20T22:20:12.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:12 smithi194 ceph-mon[25957]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:20:12.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:12 smithi194 ceph-mon[25957]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:20:12.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:12 smithi194 ceph-mon[25957]: Reconfiguring node-exporter.smithi037 (dependencies changed)... 2024-09-20T22:20:12.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:12 smithi194 ceph-mon[25957]: Deploying daemon node-exporter.smithi037 on smithi037 2024-09-20T22:20:13.806 INFO:teuthology.orchestra.run.smithi037.stdout: 5 72283 13.61 MB/sec execute 31 sec latency 21.845 ms 2024-09-20T22:20:14.667 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:14 smithi037 ceph-mon[23718]: pgmap v24: 97 pgs: 97 active+clean; 349 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.8 MiB/s wr, 1.01k op/s 2024-09-20T22:20:14.806 INFO:teuthology.orchestra.run.smithi037.stdout: 5 72825 13.37 MB/sec execute 32 sec latency 37.956 ms 2024-09-20T22:20:14.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:14 smithi194 ceph-mon[25957]: pgmap v24: 97 pgs: 97 active+clean; 349 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.8 MiB/s wr, 1.01k op/s 2024-09-20T22:20:15.807 INFO:teuthology.orchestra.run.smithi037.stdout: 5 73318 13.43 MB/sec execute 33 sec latency 34.349 ms 2024-09-20T22:20:16.695 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:16 smithi037 ceph-mon[23718]: pgmap v25: 97 pgs: 97 active+clean; 349 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 7.1 MiB/s wr, 661 op/s 2024-09-20T22:20:16.695 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:16 smithi037 ceph-mon[23718]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:20:16.807 INFO:teuthology.orchestra.run.smithi037.stdout: 5 74065 13.75 MB/sec execute 34 sec latency 21.890 ms 2024-09-20T22:20:16.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:16 smithi194 ceph-mon[25957]: pgmap v25: 97 pgs: 97 active+clean; 349 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 7.1 MiB/s wr, 661 op/s 2024-09-20T22:20:16.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:16 smithi194 ceph-mon[25957]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:20:17.807 INFO:teuthology.orchestra.run.smithi037.stdout: 5 74780 14.13 MB/sec execute 35 sec latency 29.842 ms 2024-09-20T22:20:18.706 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:18 smithi037 ceph-mon[23718]: pgmap v26: 97 pgs: 97 active+clean; 349 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 7.1 MiB/s wr, 661 op/s 2024-09-20T22:20:18.807 INFO:teuthology.orchestra.run.smithi037.stdout: 5 75342 14.02 MB/sec execute 36 sec latency 40.422 ms 2024-09-20T22:20:18.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:18 smithi194 ceph-mon[25957]: pgmap v26: 97 pgs: 97 active+clean; 349 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 7.1 MiB/s wr, 661 op/s 2024-09-20T22:20:19.807 INFO:teuthology.orchestra.run.smithi037.stdout: 5 75651 13.81 MB/sec execute 37 sec latency 65.465 ms 2024-09-20T22:20:20.562 INFO:teuthology.orchestra.run.smithi037.stdout:true 2024-09-20T22:20:20.807 INFO:teuthology.orchestra.run.smithi037.stdout: 5 76040 13.70 MB/sec execute 38 sec latency 41.923 ms 2024-09-20T22:20:20.996 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:20 smithi037 ceph-mon[23718]: pgmap v27: 97 pgs: 97 active+clean; 333 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 11 MiB/s wr, 1.00k op/s 2024-09-20T22:20:20.996 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:20 smithi037 ceph-mon[23718]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:20:20.996 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:20 smithi037 ceph-mon[23718]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:20:20.996 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:20 smithi037 ceph-mon[23718]: Reconfiguring prometheus.smithi037 (dependencies changed)... 2024-09-20T22:20:21.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:20 smithi194 ceph-mon[25957]: pgmap v27: 97 pgs: 97 active+clean; 333 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 11 MiB/s wr, 1.00k op/s 2024-09-20T22:20:21.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:20 smithi194 ceph-mon[25957]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:20:21.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:20 smithi194 ceph-mon[25957]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:20:21.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:20 smithi194 ceph-mon[25957]: Reconfiguring prometheus.smithi037 (dependencies changed)... 2024-09-20T22:20:21.681 INFO:teuthology.orchestra.run.smithi037.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T22:20:21.681 INFO:teuthology.orchestra.run.smithi037.stdout:alertmanager.smithi037 smithi037 *:9093,9094 starting - - - - 2024-09-20T22:20:21.681 INFO:teuthology.orchestra.run.smithi037.stdout:crash.smithi037 smithi037 running (9m) 37s ago 9m 6400k - 17.2.0 e1d6a67b021e 88301022bdc8 2024-09-20T22:20:21.681 INFO:teuthology.orchestra.run.smithi037.stdout:crash.smithi194 smithi194 running (8m) 44s ago 8m 6405k - 17.2.0 e1d6a67b021e b553601d7ca5 2024-09-20T22:20:21.682 INFO:teuthology.orchestra.run.smithi037.stdout:grafana.smithi037 smithi037 *:3000 starting - - - - 2024-09-20T22:20:21.682 INFO:teuthology.orchestra.run.smithi037.stdout:mds.foofs.smithi037.sdtedi smithi037 running (3m) 37s ago 3m 34.8M - 17.2.0 e1d6a67b021e 0a7898d25b8d 2024-09-20T22:20:21.682 INFO:teuthology.orchestra.run.smithi037.stdout:mds.foofs.smithi194.lkroso smithi194 running (3m) 44s ago 3m 14.2M - 17.2.0 e1d6a67b021e 17736de30008 2024-09-20T22:20:21.682 INFO:teuthology.orchestra.run.smithi037.stdout:mgr.smithi037.lykblh smithi037 *:9283 running (10m) 37s ago 10m 139M - 17.2.0 e1d6a67b021e 3ef1f99a6ba9 2024-09-20T22:20:21.682 INFO:teuthology.orchestra.run.smithi037.stdout:mgr.smithi194.bkcglt smithi194 *:8443,9283 running (71s) 44s ago 8m 478M - 19.1.1-367-g9e2ac70b cc9251b744a3 e0af2512175a 2024-09-20T22:20:21.682 INFO:teuthology.orchestra.run.smithi037.stdout:mon.smithi037 smithi037 running (10m) 37s ago 10m 49.5M 2048M 17.2.0 e1d6a67b021e 71f591122786 2024-09-20T22:20:21.682 INFO:teuthology.orchestra.run.smithi037.stdout:mon.smithi194 smithi194 running (8m) 44s ago 8m 42.5M 2048M 17.2.0 e1d6a67b021e 0b8814ae6c66 2024-09-20T22:20:21.682 INFO:teuthology.orchestra.run.smithi037.stdout:nfs.foo.0.0.smithi037.spabdc smithi037 *:2049 running (2m) 37s ago 2m 253M - 4.0 e1d6a67b021e 858ff9d558a2 2024-09-20T22:20:21.682 INFO:teuthology.orchestra.run.smithi037.stdout:nfs.foo.1.0.smithi194.qzjyej smithi194 *:2049 running (2m) 44s ago 2m 77.7M - 4.0 e1d6a67b021e c07fe0eb7c75 2024-09-20T22:20:21.682 INFO:teuthology.orchestra.run.smithi037.stdout:node-exporter.smithi037 smithi037 *:9100 starting - - - - 2024-09-20T22:20:21.682 INFO:teuthology.orchestra.run.smithi037.stdout:node-exporter.smithi194 smithi194 *:9100 running (8m) 44s ago 8m 21.0M - 1.3.1 1dbe0e931976 3a80a2ef6678 2024-09-20T22:20:21.682 INFO:teuthology.orchestra.run.smithi037.stdout:osd.0 smithi037 running (7m) 37s ago 7m 256M 1937M 17.2.0 e1d6a67b021e e54e84fd5030 2024-09-20T22:20:21.682 INFO:teuthology.orchestra.run.smithi037.stdout:osd.1 smithi037 running (7m) 37s ago 7m 193M 1937M 17.2.0 e1d6a67b021e 09647c133032 2024-09-20T22:20:21.682 INFO:teuthology.orchestra.run.smithi037.stdout:osd.2 smithi037 running (6m) 37s ago 6m 158M 1937M 17.2.0 e1d6a67b021e 6c15b1175256 2024-09-20T22:20:21.682 INFO:teuthology.orchestra.run.smithi037.stdout:osd.3 smithi037 running (6m) 37s ago 6m 208M 1937M 17.2.0 e1d6a67b021e 3a22952c6475 2024-09-20T22:20:21.683 INFO:teuthology.orchestra.run.smithi037.stdout:osd.4 smithi194 running (5m) 44s ago 5m 224M 2704M 17.2.0 e1d6a67b021e fa107be8915c 2024-09-20T22:20:21.683 INFO:teuthology.orchestra.run.smithi037.stdout:osd.5 smithi194 running (5m) 44s ago 5m 216M 2704M 17.2.0 e1d6a67b021e 290b21cbd131 2024-09-20T22:20:21.683 INFO:teuthology.orchestra.run.smithi037.stdout:osd.6 smithi194 running (5m) 44s ago 5m 203M 2704M 17.2.0 e1d6a67b021e 27f0720639ed 2024-09-20T22:20:21.683 INFO:teuthology.orchestra.run.smithi037.stdout:osd.7 smithi194 running (4m) 44s ago 4m 307M 2704M 17.2.0 e1d6a67b021e 62d163f4fd92 2024-09-20T22:20:21.683 INFO:teuthology.orchestra.run.smithi037.stdout:prometheus.smithi037 smithi037 *:9095 running (8m) 37s ago 8m 64.0M - 2.33.4 514e6a882f6e 32020a1cc2f4 2024-09-20T22:20:21.807 INFO:teuthology.orchestra.run.smithi037.stdout: 5 76544 13.63 MB/sec execute 39 sec latency 50.365 ms 2024-09-20T22:20:22.019 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:21 smithi037 ceph-mon[23718]: Deploying daemon prometheus.smithi037 on smithi037 2024-09-20T22:20:22.019 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:21 smithi037 ceph-mon[23718]: from='client.24515 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:20:22.332 INFO:teuthology.orchestra.run.smithi037.stdout:{ 2024-09-20T22:20:22.332 INFO:teuthology.orchestra.run.smithi037.stdout: "mon": { 2024-09-20T22:20:22.332 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T22:20:22.332 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:20:22.332 INFO:teuthology.orchestra.run.smithi037.stdout: "mgr": { 2024-09-20T22:20:22.332 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-20T22:20:22.332 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 1 2024-09-20T22:20:22.332 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:20:22.332 INFO:teuthology.orchestra.run.smithi037.stdout: "osd": { 2024-09-20T22:20:22.332 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-20T22:20:22.333 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:20:22.333 INFO:teuthology.orchestra.run.smithi037.stdout: "mds": { 2024-09-20T22:20:22.333 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T22:20:22.333 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:20:22.333 INFO:teuthology.orchestra.run.smithi037.stdout: "overall": { 2024-09-20T22:20:22.333 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-09-20T22:20:22.333 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 1 2024-09-20T22:20:22.333 INFO:teuthology.orchestra.run.smithi037.stdout: } 2024-09-20T22:20:22.333 INFO:teuthology.orchestra.run.smithi037.stdout:} 2024-09-20T22:20:22.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:21 smithi194 ceph-mon[25957]: Deploying daemon prometheus.smithi037 on smithi037 2024-09-20T22:20:22.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:21 smithi194 ceph-mon[25957]: from='client.24515 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:20:22.807 INFO:teuthology.orchestra.run.smithi037.stdout: 5 77028 13.52 MB/sec execute 40 sec latency 31.631 ms 2024-09-20T22:20:22.931 INFO:teuthology.orchestra.run.smithi037.stdout:{ 2024-09-20T22:20:22.931 INFO:teuthology.orchestra.run.smithi037.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5", 2024-09-20T22:20:22.931 INFO:teuthology.orchestra.run.smithi037.stdout: "in_progress": true, 2024-09-20T22:20:22.931 INFO:teuthology.orchestra.run.smithi037.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-20T22:20:22.931 INFO:teuthology.orchestra.run.smithi037.stdout: "services_complete": [], 2024-09-20T22:20:22.931 INFO:teuthology.orchestra.run.smithi037.stdout: "progress": "1/23 daemons upgraded", 2024-09-20T22:20:22.931 INFO:teuthology.orchestra.run.smithi037.stdout: "message": "", 2024-09-20T22:20:22.931 INFO:teuthology.orchestra.run.smithi037.stdout: "is_paused": false 2024-09-20T22:20:22.931 INFO:teuthology.orchestra.run.smithi037.stdout:} 2024-09-20T22:20:23.032 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:22 smithi037 ceph-mon[23718]: from='client.14732 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:20:23.032 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:22 smithi037 ceph-mon[23718]: pgmap v28: 97 pgs: 97 active+clean; 333 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 7.1 MiB/s wr, 690 op/s 2024-09-20T22:20:23.032 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:22 smithi037 ceph-mon[23718]: from='client.14736 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:20:23.032 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:22 smithi037 ceph-mon[23718]: from='client.? 172.21.15.37:0/3159523106' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:20:23.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:22 smithi194 ceph-mon[25957]: from='client.14732 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:20:23.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:22 smithi194 ceph-mon[25957]: pgmap v28: 97 pgs: 97 active+clean; 333 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 7.1 MiB/s wr, 690 op/s 2024-09-20T22:20:23.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:22 smithi194 ceph-mon[25957]: from='client.14736 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:20:23.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:22 smithi194 ceph-mon[25957]: from='client.? 172.21.15.37:0/3159523106' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:20:23.641 INFO:teuthology.orchestra.run.smithi037.stdout:HEALTH_OK 2024-09-20T22:20:23.807 INFO:teuthology.orchestra.run.smithi037.stdout: 5 77528 13.66 MB/sec execute 41 sec latency 63.178 ms 2024-09-20T22:20:24.044 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:23 smithi037 ceph-mon[23718]: from='client.24529 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:20:24.044 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:23 smithi037 ceph-mon[23718]: from='client.? 172.21.15.37:0/3513181373' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T22:20:24.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:23 smithi194 ceph-mon[25957]: from='client.24529 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:20:24.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:23 smithi194 ceph-mon[25957]: from='client.? 172.21.15.37:0/3513181373' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T22:20:24.807 INFO:teuthology.orchestra.run.smithi037.stdout: 5 78112 13.89 MB/sec execute 42 sec latency 25.363 ms 2024-09-20T22:20:25.086 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:24 smithi037 ceph-mon[23718]: pgmap v29: 97 pgs: 97 active+clean; 319 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 9.7 MiB/s wr, 975 op/s 2024-09-20T22:20:25.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:24 smithi194 ceph-mon[25957]: pgmap v29: 97 pgs: 97 active+clean; 319 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 9.7 MiB/s wr, 975 op/s 2024-09-20T22:20:25.807 INFO:teuthology.orchestra.run.smithi037.stdout: 5 78598 13.87 MB/sec execute 43 sec latency 25.514 ms 2024-09-20T22:20:26.807 INFO:teuthology.orchestra.run.smithi037.stdout: 5 79057 13.74 MB/sec execute 44 sec latency 98.679 ms 2024-09-20T22:20:27.101 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:26 smithi037 ceph-mon[23718]: pgmap v30: 97 pgs: 97 active+clean; 319 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 6.1 MiB/s wr, 628 op/s 2024-09-20T22:20:27.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:26 smithi194 ceph-mon[25957]: pgmap v30: 97 pgs: 97 active+clean; 319 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 6.1 MiB/s wr, 628 op/s 2024-09-20T22:20:27.807 INFO:teuthology.orchestra.run.smithi037.stdout: 5 79401 13.58 MB/sec execute 45 sec latency 51.540 ms 2024-09-20T22:20:28.807 INFO:teuthology.orchestra.run.smithi037.stdout: 5 79758 13.44 MB/sec execute 46 sec latency 67.068 ms 2024-09-20T22:20:29.119 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:28 smithi037 ceph-mon[23718]: pgmap v31: 97 pgs: 97 active+clean; 319 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 6.1 MiB/s wr, 628 op/s 2024-09-20T22:20:29.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:28 smithi194 ceph-mon[25957]: pgmap v31: 97 pgs: 97 active+clean; 319 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 6.1 MiB/s wr, 628 op/s 2024-09-20T22:20:29.807 INFO:teuthology.orchestra.run.smithi037.stdout: 5 80169 13.35 MB/sec execute 47 sec latency 53.464 ms 2024-09-20T22:20:30.807 INFO:teuthology.orchestra.run.smithi037.stdout: 5 80596 13.27 MB/sec execute 48 sec latency 54.595 ms 2024-09-20T22:20:31.158 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:30 smithi037 ceph-mon[23718]: pgmap v32: 97 pgs: 97 active+clean; 338 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.6 MiB/s wr, 915 op/s 2024-09-20T22:20:31.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:30 smithi194 ceph-mon[25957]: pgmap v32: 97 pgs: 97 active+clean; 338 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.6 MiB/s wr, 915 op/s 2024-09-20T22:20:31.807 INFO:teuthology.orchestra.run.smithi037.stdout: 5 81019 13.36 MB/sec execute 49 sec latency 77.950 ms 2024-09-20T22:20:32.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:32 smithi037 ceph-mon[23718]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:20:32.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:32 smithi194 ceph-mon[25957]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:20:32.807 INFO:teuthology.orchestra.run.smithi037.stdout: 5 81498 13.52 MB/sec execute 50 sec latency 57.716 ms 2024-09-20T22:20:33.189 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:33 smithi037 ceph-mon[23718]: pgmap v33: 97 pgs: 97 active+clean; 338 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.9 KiB/s rd, 6.0 MiB/s wr, 572 op/s 2024-09-20T22:20:33.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:33 smithi194 ceph-mon[25957]: pgmap v33: 97 pgs: 97 active+clean; 338 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.9 KiB/s rd, 6.0 MiB/s wr, 572 op/s 2024-09-20T22:20:33.807 INFO:teuthology.orchestra.run.smithi037.stdout: 5 81926 13.51 MB/sec execute 51 sec latency 53.376 ms 2024-09-20T22:20:34.807 INFO:teuthology.orchestra.run.smithi037.stdout: 5 82371 13.36 MB/sec execute 52 sec latency 55.456 ms 2024-09-20T22:20:35.206 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:35 smithi037 ceph-mon[23718]: pgmap v34: 97 pgs: 97 active+clean; 326 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.3 MiB/s wr, 811 op/s 2024-09-20T22:20:35.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:35 smithi194 ceph-mon[25957]: pgmap v34: 97 pgs: 97 active+clean; 326 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.3 MiB/s wr, 811 op/s 2024-09-20T22:20:35.807 INFO:teuthology.orchestra.run.smithi037.stdout: 5 82726 13.25 MB/sec execute 53 sec latency 65.696 ms 2024-09-20T22:20:36.814 INFO:teuthology.orchestra.run.smithi037.stdout: 5 83284 13.25 MB/sec execute 54 sec latency 43.060 ms 2024-09-20T22:20:37.219 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:37 smithi037 ceph-mon[23718]: pgmap v35: 97 pgs: 97 active+clean; 326 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.7 MiB/s wr, 525 op/s 2024-09-20T22:20:37.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:37 smithi194 ceph-mon[25957]: pgmap v35: 97 pgs: 97 active+clean; 326 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.7 MiB/s wr, 525 op/s 2024-09-20T22:20:37.814 INFO:teuthology.orchestra.run.smithi037.stdout: 5 84119 13.31 MB/sec execute 55 sec latency 27.138 ms 2024-09-20T22:20:38.814 INFO:teuthology.orchestra.run.smithi037.stdout: 5 84879 13.63 MB/sec execute 56 sec latency 23.525 ms 2024-09-20T22:20:39.236 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:39 smithi037 ceph-mon[23718]: pgmap v36: 97 pgs: 97 active+clean; 326 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.7 MiB/s wr, 525 op/s 2024-09-20T22:20:39.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:39 smithi194 ceph-mon[25957]: pgmap v36: 97 pgs: 97 active+clean; 326 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.7 MiB/s wr, 525 op/s 2024-09-20T22:20:39.814 INFO:teuthology.orchestra.run.smithi037.stdout: 5 85459 13.76 MB/sec execute 57 sec latency 25.945 ms 2024-09-20T22:20:40.814 INFO:teuthology.orchestra.run.smithi037.stdout: 5 85789 13.60 MB/sec execute 58 sec latency 30.142 ms 2024-09-20T22:20:41.257 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:41 smithi037 ceph-mon[23718]: pgmap v37: 97 pgs: 97 active+clean; 323 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.4 MiB/s wr, 814 op/s 2024-09-20T22:20:41.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:41 smithi194 ceph-mon[25957]: pgmap v37: 97 pgs: 97 active+clean; 323 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.4 MiB/s wr, 814 op/s 2024-09-20T22:20:41.814 INFO:teuthology.orchestra.run.smithi037.stdout: 5 86190 13.49 MB/sec execute 59 sec latency 24.247 ms 2024-09-20T22:20:42.814 INFO:teuthology.orchestra.run.smithi037.stdout: 5 86616 13.45 MB/sec execute 60 sec latency 36.438 ms 2024-09-20T22:20:43.035 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:43 smithi037 ceph-mon[23718]: pgmap v38: 97 pgs: 97 active+clean; 323 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.0 MiB/s wr, 527 op/s 2024-09-20T22:20:43.035 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:43 smithi037 ceph-mon[23718]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:20:43.035 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:43 smithi037 ceph-mon[23718]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:20:43.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:43 smithi194 ceph-mon[25957]: pgmap v38: 97 pgs: 97 active+clean; 323 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.0 MiB/s wr, 527 op/s 2024-09-20T22:20:43.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:43 smithi194 ceph-mon[25957]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:20:43.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:43 smithi194 ceph-mon[25957]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:20:43.814 INFO:teuthology.orchestra.run.smithi037.stdout: 5 87096 13.38 MB/sec execute 61 sec latency 37.271 ms 2024-09-20T22:20:44.048 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:44 smithi037 ceph-mon[23718]: Reconfiguring node-exporter.smithi194 (dependencies changed)... 2024-09-20T22:20:44.048 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:44 smithi037 ceph-mon[23718]: Deploying daemon node-exporter.smithi194 on smithi194 2024-09-20T22:20:44.246 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:44 smithi194 ceph-mon[25957]: Reconfiguring node-exporter.smithi194 (dependencies changed)... 2024-09-20T22:20:44.246 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:44 smithi194 ceph-mon[25957]: Deploying daemon node-exporter.smithi194 on smithi194 2024-09-20T22:20:44.814 INFO:teuthology.orchestra.run.smithi037.stdout: 5 87808 13.43 MB/sec execute 62 sec latency 34.030 ms 2024-09-20T22:20:45.054 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:45 smithi037 ceph-mon[23718]: pgmap v39: 97 pgs: 97 active+clean; 333 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 8.8 MiB/s wr, 858 op/s 2024-09-20T22:20:45.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:45 smithi194 ceph-mon[25957]: pgmap v39: 97 pgs: 97 active+clean; 333 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 8.8 MiB/s wr, 858 op/s 2024-09-20T22:20:45.814 INFO:teuthology.orchestra.run.smithi037.stdout: 5 88345 13.56 MB/sec execute 63 sec latency 30.098 ms 2024-09-20T22:20:46.814 INFO:teuthology.orchestra.run.smithi037.stdout: 5 88763 13.61 MB/sec execute 64 sec latency 33.124 ms 2024-09-20T22:20:47.084 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:47 smithi037 ceph-mon[23718]: pgmap v40: 97 pgs: 97 active+clean; 333 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 8.6 KiB/s rd, 6.5 MiB/s wr, 619 op/s 2024-09-20T22:20:47.084 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:47 smithi037 ceph-mon[23718]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:20:47.277 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:47 smithi194 ceph-mon[25957]: pgmap v40: 97 pgs: 97 active+clean; 333 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 8.6 KiB/s rd, 6.5 MiB/s wr, 619 op/s 2024-09-20T22:20:47.277 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:47 smithi194 ceph-mon[25957]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:20:47.814 INFO:teuthology.orchestra.run.smithi037.stdout: 5 89094 13.56 MB/sec execute 65 sec latency 96.933 ms 2024-09-20T22:20:48.521 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:48 smithi037 ceph-mon[23718]: pgmap v41: 97 pgs: 97 active+clean; 333 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 8.6 KiB/s rd, 6.5 MiB/s wr, 619 op/s 2024-09-20T22:20:48.521 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:48 smithi037 ceph-mon[23718]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:20:48.521 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:48 smithi037 ceph-mon[23718]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:20:48.521 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:48 smithi037 ceph-mon[23718]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-20T22:20:48.521 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:48 smithi037 ceph-mon[23718]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-20T22:20:48.521 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:48 smithi037 ceph-mon[23718]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi037.front.sepia.ceph.com:9093"}]: dispatch 2024-09-20T22:20:48.521 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:48 smithi037 ceph-mon[23718]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi037.front.sepia.ceph.com:9093"}]: dispatch 2024-09-20T22:20:48.522 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:48 smithi037 ceph-mon[23718]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:20:48.522 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:48 smithi037 ceph-mon[23718]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-20T22:20:48.522 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:48 smithi037 ceph-mon[23718]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-20T22:20:48.522 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:48 smithi037 ceph-mon[23718]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi037.front.sepia.ceph.com:3000"}]: dispatch 2024-09-20T22:20:48.522 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:48 smithi037 ceph-mon[23718]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi037.front.sepia.ceph.com:3000"}]: dispatch 2024-09-20T22:20:48.522 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:48 smithi037 ceph-mon[23718]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:20:48.522 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:48 smithi037 ceph-mon[23718]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-20T22:20:48.522 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:48 smithi037 ceph-mon[23718]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-20T22:20:48.522 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:48 smithi037 ceph-mon[23718]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi037.front.sepia.ceph.com:9095"}]: dispatch 2024-09-20T22:20:48.522 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:48 smithi037 ceph-mon[23718]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi037.front.sepia.ceph.com:9095"}]: dispatch 2024-09-20T22:20:48.522 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:48 smithi037 ceph-mon[23718]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:20:48.522 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:48 smithi037 ceph-mon[23718]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:20:48.522 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:48 smithi037 ceph-mon[23718]: Upgrade: Need to upgrade myself (mgr.smithi194.bkcglt) 2024-09-20T22:20:48.522 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:48 smithi037 ceph-mon[23718]: Upgrade: Need to upgrade myself (mgr.smithi194.bkcglt) 2024-09-20T22:20:48.746 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:48 smithi194 ceph-mon[25957]: pgmap v41: 97 pgs: 97 active+clean; 333 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 8.6 KiB/s rd, 6.5 MiB/s wr, 619 op/s 2024-09-20T22:20:48.746 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:48 smithi194 ceph-mon[25957]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:20:48.747 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:48 smithi194 ceph-mon[25957]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:20:48.747 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:48 smithi194 ceph-mon[25957]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-20T22:20:48.747 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:48 smithi194 ceph-mon[25957]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-20T22:20:48.747 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:48 smithi194 ceph-mon[25957]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi037.front.sepia.ceph.com:9093"}]: dispatch 2024-09-20T22:20:48.747 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:48 smithi194 ceph-mon[25957]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi037.front.sepia.ceph.com:9093"}]: dispatch 2024-09-20T22:20:48.747 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:48 smithi194 ceph-mon[25957]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:20:48.747 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:48 smithi194 ceph-mon[25957]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-20T22:20:48.747 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:48 smithi194 ceph-mon[25957]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-20T22:20:48.747 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:48 smithi194 ceph-mon[25957]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi037.front.sepia.ceph.com:3000"}]: dispatch 2024-09-20T22:20:48.747 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:48 smithi194 ceph-mon[25957]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi037.front.sepia.ceph.com:3000"}]: dispatch 2024-09-20T22:20:48.747 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:48 smithi194 ceph-mon[25957]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:20:48.747 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:48 smithi194 ceph-mon[25957]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-20T22:20:48.747 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:48 smithi194 ceph-mon[25957]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-20T22:20:48.747 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:48 smithi194 ceph-mon[25957]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi037.front.sepia.ceph.com:9095"}]: dispatch 2024-09-20T22:20:48.748 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:48 smithi194 ceph-mon[25957]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi037.front.sepia.ceph.com:9095"}]: dispatch 2024-09-20T22:20:48.748 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:48 smithi194 ceph-mon[25957]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:20:48.748 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:48 smithi194 ceph-mon[25957]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:20:48.748 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:48 smithi194 ceph-mon[25957]: Upgrade: Need to upgrade myself (mgr.smithi194.bkcglt) 2024-09-20T22:20:48.748 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:48 smithi194 ceph-mon[25957]: Upgrade: Need to upgrade myself (mgr.smithi194.bkcglt) 2024-09-20T22:20:48.815 INFO:teuthology.orchestra.run.smithi037.stdout: 5 89440 13.42 MB/sec execute 66 sec latency 61.387 ms 2024-09-20T22:20:49.814 INFO:teuthology.orchestra.run.smithi037.stdout: 5 89676 13.29 MB/sec execute 67 sec latency 67.248 ms 2024-09-20T22:20:50.804 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:50 smithi037 ceph-mon[23718]: pgmap v42: 97 pgs: 97 active+clean; 333 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.7 MiB/s wr, 907 op/s 2024-09-20T22:20:50.804 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:50 smithi037 ceph-mon[23718]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:20:50.804 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:50 smithi037 ceph-mon[23718]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi037.lykblh", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-20T22:20:50.804 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:50 smithi037 ceph-mon[23718]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi037.lykblh", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-20T22:20:50.804 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:50 smithi037 ceph-mon[23718]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-20T22:20:50.805 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:50 smithi037 ceph-mon[23718]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:20:50.814 INFO:teuthology.orchestra.run.smithi037.stdout: 5 89958 13.15 MB/sec execute 68 sec latency 88.846 ms 2024-09-20T22:20:50.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:50 smithi194 ceph-mon[25957]: pgmap v42: 97 pgs: 97 active+clean; 333 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.7 MiB/s wr, 907 op/s 2024-09-20T22:20:50.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:50 smithi194 ceph-mon[25957]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:20:50.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:50 smithi194 ceph-mon[25957]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi037.lykblh", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-20T22:20:50.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:50 smithi194 ceph-mon[25957]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi037.lykblh", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-20T22:20:50.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:50 smithi194 ceph-mon[25957]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-20T22:20:50.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:50 smithi194 ceph-mon[25957]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:20:51.809 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:51 smithi037 ceph-mon[23718]: Upgrade: Updating mgr.smithi037.lykblh 2024-09-20T22:20:51.809 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:51 smithi037 ceph-mon[23718]: Deploying daemon mgr.smithi037.lykblh on smithi037 2024-09-20T22:20:51.814 INFO:teuthology.orchestra.run.smithi037.stdout: 5 90479 13.16 MB/sec execute 69 sec latency 60.626 ms 2024-09-20T22:20:51.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:51 smithi194 ceph-mon[25957]: Upgrade: Updating mgr.smithi037.lykblh 2024-09-20T22:20:51.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:51 smithi194 ceph-mon[25957]: Deploying daemon mgr.smithi037.lykblh on smithi037 2024-09-20T22:20:52.589 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:52 smithi037 ceph-mon[23718]: pgmap v43: 97 pgs: 97 active+clean; 333 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 6.9 MiB/s wr, 619 op/s 2024-09-20T22:20:52.815 INFO:teuthology.orchestra.run.smithi037.stdout: 5 90970 13.13 MB/sec execute 70 sec latency 50.148 ms 2024-09-20T22:20:52.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:52 smithi194 ceph-mon[25957]: pgmap v43: 97 pgs: 97 active+clean; 333 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 6.9 MiB/s wr, 619 op/s 2024-09-20T22:20:53.815 INFO:teuthology.orchestra.run.smithi037.stdout: 5 91375 13.08 MB/sec execute 71 sec latency 47.767 ms 2024-09-20T22:20:54.138 INFO:teuthology.orchestra.run.smithi037.stdout:true 2024-09-20T22:20:54.630 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:54 smithi037 ceph-mon[23718]: pgmap v44: 97 pgs: 97 active+clean; 325 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 8.7 MiB/s wr, 846 op/s 2024-09-20T22:20:54.815 INFO:teuthology.orchestra.run.smithi037.stdout: 5 91773 13.14 MB/sec execute 72 sec latency 39.146 ms 2024-09-20T22:20:54.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:54 smithi194 ceph-mon[25957]: pgmap v44: 97 pgs: 97 active+clean; 325 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 8.7 MiB/s wr, 846 op/s 2024-09-20T22:20:55.101 INFO:teuthology.orchestra.run.smithi037.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T22:20:55.101 INFO:teuthology.orchestra.run.smithi037.stdout:alertmanager.smithi037 smithi037 *:9093,9094 starting - - - - 2024-09-20T22:20:55.101 INFO:teuthology.orchestra.run.smithi037.stdout:crash.smithi037 smithi037 running (9m) 70s ago 9m 6400k - 17.2.0 e1d6a67b021e 88301022bdc8 2024-09-20T22:20:55.101 INFO:teuthology.orchestra.run.smithi037.stdout:crash.smithi194 smithi194 running (9m) 78s ago 9m 6405k - 17.2.0 e1d6a67b021e b553601d7ca5 2024-09-20T22:20:55.101 INFO:teuthology.orchestra.run.smithi037.stdout:grafana.smithi037 smithi037 *:3000 starting - - - - 2024-09-20T22:20:55.101 INFO:teuthology.orchestra.run.smithi037.stdout:mds.foofs.smithi037.sdtedi smithi037 running (3m) 70s ago 3m 34.8M - 17.2.0 e1d6a67b021e 0a7898d25b8d 2024-09-20T22:20:55.101 INFO:teuthology.orchestra.run.smithi037.stdout:mds.foofs.smithi194.lkroso smithi194 running (3m) 78s ago 3m 14.2M - 17.2.0 e1d6a67b021e 17736de30008 2024-09-20T22:20:55.101 INFO:teuthology.orchestra.run.smithi037.stdout:mgr.smithi037.lykblh smithi037 *:9283 running (10m) 70s ago 10m 139M - 17.2.0 e1d6a67b021e 3ef1f99a6ba9 2024-09-20T22:20:55.101 INFO:teuthology.orchestra.run.smithi037.stdout:mgr.smithi194.bkcglt smithi194 *:8443,9283 running (104s) 78s ago 9m 478M - 19.1.1-367-g9e2ac70b cc9251b744a3 e0af2512175a 2024-09-20T22:20:55.101 INFO:teuthology.orchestra.run.smithi037.stdout:mon.smithi037 smithi037 running (11m) 70s ago 11m 49.5M 2048M 17.2.0 e1d6a67b021e 71f591122786 2024-09-20T22:20:55.102 INFO:teuthology.orchestra.run.smithi037.stdout:mon.smithi194 smithi194 running (8m) 78s ago 8m 42.5M 2048M 17.2.0 e1d6a67b021e 0b8814ae6c66 2024-09-20T22:20:55.102 INFO:teuthology.orchestra.run.smithi037.stdout:nfs.foo.0.0.smithi037.spabdc smithi037 *:2049 running (3m) 70s ago 3m 253M - 4.0 e1d6a67b021e 858ff9d558a2 2024-09-20T22:20:55.102 INFO:teuthology.orchestra.run.smithi037.stdout:nfs.foo.1.0.smithi194.qzjyej smithi194 *:2049 running (3m) 78s ago 3m 77.7M - 4.0 e1d6a67b021e c07fe0eb7c75 2024-09-20T22:20:55.102 INFO:teuthology.orchestra.run.smithi037.stdout:node-exporter.smithi037 smithi037 *:9100 starting - - - - 2024-09-20T22:20:55.102 INFO:teuthology.orchestra.run.smithi037.stdout:node-exporter.smithi194 smithi194 *:9100 starting - - - - 2024-09-20T22:20:55.102 INFO:teuthology.orchestra.run.smithi037.stdout:osd.0 smithi037 running (8m) 70s ago 8m 256M 1937M 17.2.0 e1d6a67b021e e54e84fd5030 2024-09-20T22:20:55.102 INFO:teuthology.orchestra.run.smithi037.stdout:osd.1 smithi037 running (7m) 70s ago 7m 193M 1937M 17.2.0 e1d6a67b021e 09647c133032 2024-09-20T22:20:55.102 INFO:teuthology.orchestra.run.smithi037.stdout:osd.2 smithi037 running (7m) 70s ago 7m 158M 1937M 17.2.0 e1d6a67b021e 6c15b1175256 2024-09-20T22:20:55.102 INFO:teuthology.orchestra.run.smithi037.stdout:osd.3 smithi037 running (6m) 70s ago 6m 208M 1937M 17.2.0 e1d6a67b021e 3a22952c6475 2024-09-20T22:20:55.102 INFO:teuthology.orchestra.run.smithi037.stdout:osd.4 smithi194 running (6m) 78s ago 6m 224M 2704M 17.2.0 e1d6a67b021e fa107be8915c 2024-09-20T22:20:55.102 INFO:teuthology.orchestra.run.smithi037.stdout:osd.5 smithi194 running (6m) 78s ago 6m 216M 2704M 17.2.0 e1d6a67b021e 290b21cbd131 2024-09-20T22:20:55.102 INFO:teuthology.orchestra.run.smithi037.stdout:osd.6 smithi194 running (5m) 78s ago 5m 203M 2704M 17.2.0 e1d6a67b021e 27f0720639ed 2024-09-20T22:20:55.102 INFO:teuthology.orchestra.run.smithi037.stdout:osd.7 smithi194 running (5m) 78s ago 5m 307M 2704M 17.2.0 e1d6a67b021e 62d163f4fd92 2024-09-20T22:20:55.103 INFO:teuthology.orchestra.run.smithi037.stdout:prometheus.smithi037 smithi037 *:9095 starting - - - - 2024-09-20T22:20:55.655 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:55 smithi037 ceph-mon[23718]: from='client.14752 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:20:55.655 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:55 smithi037 ceph-mon[23718]: from='client.14756 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:20:55.727 INFO:teuthology.orchestra.run.smithi037.stdout:{ 2024-09-20T22:20:55.727 INFO:teuthology.orchestra.run.smithi037.stdout: "mon": { 2024-09-20T22:20:55.727 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T22:20:55.728 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:20:55.728 INFO:teuthology.orchestra.run.smithi037.stdout: "mgr": { 2024-09-20T22:20:55.728 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-20T22:20:55.728 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 1 2024-09-20T22:20:55.728 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:20:55.728 INFO:teuthology.orchestra.run.smithi037.stdout: "osd": { 2024-09-20T22:20:55.728 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-20T22:20:55.728 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:20:55.728 INFO:teuthology.orchestra.run.smithi037.stdout: "mds": { 2024-09-20T22:20:55.728 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T22:20:55.728 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:20:55.728 INFO:teuthology.orchestra.run.smithi037.stdout: "overall": { 2024-09-20T22:20:55.728 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-09-20T22:20:55.728 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 1 2024-09-20T22:20:55.729 INFO:teuthology.orchestra.run.smithi037.stdout: } 2024-09-20T22:20:55.729 INFO:teuthology.orchestra.run.smithi037.stdout:} 2024-09-20T22:20:55.815 INFO:teuthology.orchestra.run.smithi037.stdout: 5 92125 13.15 MB/sec execute 73 sec latency 49.841 ms 2024-09-20T22:20:55.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:55 smithi194 ceph-mon[25957]: from='client.14752 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:20:55.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:55 smithi194 ceph-mon[25957]: from='client.14756 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:20:56.206 INFO:teuthology.orchestra.run.smithi037.stdout:{ 2024-09-20T22:20:56.206 INFO:teuthology.orchestra.run.smithi037.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5", 2024-09-20T22:20:56.206 INFO:teuthology.orchestra.run.smithi037.stdout: "in_progress": true, 2024-09-20T22:20:56.206 INFO:teuthology.orchestra.run.smithi037.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-20T22:20:56.206 INFO:teuthology.orchestra.run.smithi037.stdout: "services_complete": [], 2024-09-20T22:20:56.206 INFO:teuthology.orchestra.run.smithi037.stdout: "progress": "1/23 daemons upgraded", 2024-09-20T22:20:56.206 INFO:teuthology.orchestra.run.smithi037.stdout: "message": "Currently upgrading mgr daemons", 2024-09-20T22:20:56.207 INFO:teuthology.orchestra.run.smithi037.stdout: "is_paused": false 2024-09-20T22:20:56.207 INFO:teuthology.orchestra.run.smithi037.stdout:} 2024-09-20T22:20:56.675 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:56 smithi037 ceph-mon[23718]: from='client.24541 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:20:56.675 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:56 smithi037 ceph-mon[23718]: pgmap v45: 97 pgs: 97 active+clean; 325 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.0 MiB/s wr, 515 op/s 2024-09-20T22:20:56.675 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:56 smithi037 ceph-mon[23718]: from='client.? 172.21.15.37:0/1445782381' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:20:56.766 INFO:teuthology.orchestra.run.smithi037.stdout:HEALTH_OK 2024-09-20T22:20:56.815 INFO:teuthology.orchestra.run.smithi037.stdout: 5 92386 13.12 MB/sec execute 74 sec latency 52.526 ms 2024-09-20T22:20:56.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:56 smithi194 ceph-mon[25957]: from='client.24541 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:20:56.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:56 smithi194 ceph-mon[25957]: pgmap v45: 97 pgs: 97 active+clean; 325 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.0 MiB/s wr, 515 op/s 2024-09-20T22:20:56.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:56 smithi194 ceph-mon[25957]: from='client.? 172.21.15.37:0/1445782381' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:20:57.703 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:57 smithi037 ceph-mon[23718]: from='client.14768 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:20:57.704 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:57 smithi037 ceph-mon[23718]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:20:57.704 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:57 smithi037 ceph-mon[23718]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:20:57.704 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:57 smithi037 ceph-mon[23718]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:20:57.704 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:57 smithi037 ceph-mon[23718]: from='client.? 172.21.15.37:0/2533214411' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T22:20:57.704 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:57 smithi037 ceph-mon[23718]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:20:57.704 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:57 smithi037 ceph-mon[23718]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:20:57.815 INFO:teuthology.orchestra.run.smithi037.stdout: 5 92599 13.02 MB/sec execute 75 sec latency 61.548 ms 2024-09-20T22:20:57.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:57 smithi194 ceph-mon[25957]: from='client.14768 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:20:57.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:57 smithi194 ceph-mon[25957]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:20:57.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:57 smithi194 ceph-mon[25957]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:20:57.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:57 smithi194 ceph-mon[25957]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:20:57.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:57 smithi194 ceph-mon[25957]: from='client.? 172.21.15.37:0/2533214411' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T22:20:57.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:57 smithi194 ceph-mon[25957]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:20:57.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:57 smithi194 ceph-mon[25957]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:20:58.815 INFO:teuthology.orchestra.run.smithi037.stdout: 5 92850 12.90 MB/sec execute 76 sec latency 57.038 ms 2024-09-20T22:20:59.253 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:20:59 smithi037 ceph-mon[23718]: pgmap v46: 97 pgs: 97 active+clean; 325 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.0 MiB/s wr, 515 op/s 2024-09-20T22:20:59.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:20:59 smithi194 ceph-mon[25957]: pgmap v46: 97 pgs: 97 active+clean; 325 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.0 MiB/s wr, 515 op/s 2024-09-20T22:20:59.815 INFO:teuthology.orchestra.run.smithi037.stdout: 5 93107 12.76 MB/sec execute 77 sec latency 102.798 ms 2024-09-20T22:21:00.815 INFO:teuthology.orchestra.run.smithi037.stdout: 5 93367 12.68 MB/sec execute 78 sec latency 59.347 ms 2024-09-20T22:21:01.081 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:01 smithi037 ceph-mon[23718]: pgmap v47: 97 pgs: 97 active+clean; 333 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.6 MiB/s wr, 717 op/s 2024-09-20T22:21:01.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:01 smithi194 ceph-mon[25957]: pgmap v47: 97 pgs: 97 active+clean; 333 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.6 MiB/s wr, 717 op/s 2024-09-20T22:21:01.815 INFO:teuthology.orchestra.run.smithi037.stdout: 5 93659 12.59 MB/sec execute 79 sec latency 62.593 ms 2024-09-20T22:21:02.366 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:02 smithi037 ceph-mon[23718]: pgmap v48: 97 pgs: 97 active+clean; 333 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.4 MiB/s wr, 429 op/s 2024-09-20T22:21:02.366 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:02 smithi037 ceph-mon[23718]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:21:02.366 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:02 smithi037 ceph-mon[23718]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:21:02.366 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:02 smithi037 ceph-mon[23718]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:21:02.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:02 smithi194 ceph-mon[25957]: pgmap v48: 97 pgs: 97 active+clean; 333 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.4 MiB/s wr, 429 op/s 2024-09-20T22:21:02.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:02 smithi194 ceph-mon[25957]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:21:02.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:02 smithi194 ceph-mon[25957]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:21:02.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:02 smithi194 ceph-mon[25957]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:21:02.815 INFO:teuthology.orchestra.run.smithi037.stdout: 5 94064 12.55 MB/sec execute 80 sec latency 40.297 ms 2024-09-20T22:21:03.392 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:03 smithi037 ceph-mon[23718]: Standby manager daemon smithi037.lykblh restarted 2024-09-20T22:21:03.392 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:03 smithi037 ceph-mon[23718]: Standby manager daemon smithi037.lykblh started 2024-09-20T22:21:03.393 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:03 smithi037 ceph-mon[23718]: from='mgr.? 172.21.15.37:0/484483940' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi037.lykblh/crt"}]: dispatch 2024-09-20T22:21:03.393 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:03 smithi037 ceph-mon[23718]: from='mgr.? 172.21.15.37:0/484483940' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-20T22:21:03.393 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:03 smithi037 ceph-mon[23718]: from='mgr.? 172.21.15.37:0/484483940' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi037.lykblh/key"}]: dispatch 2024-09-20T22:21:03.393 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:03 smithi037 ceph-mon[23718]: from='mgr.? 172.21.15.37:0/484483940' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-20T22:21:03.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:03 smithi194 ceph-mon[25957]: Standby manager daemon smithi037.lykblh restarted 2024-09-20T22:21:03.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:03 smithi194 ceph-mon[25957]: Standby manager daemon smithi037.lykblh started 2024-09-20T22:21:03.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:03 smithi194 ceph-mon[25957]: from='mgr.? 172.21.15.37:0/484483940' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi037.lykblh/crt"}]: dispatch 2024-09-20T22:21:03.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:03 smithi194 ceph-mon[25957]: from='mgr.? 172.21.15.37:0/484483940' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-20T22:21:03.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:03 smithi194 ceph-mon[25957]: from='mgr.? 172.21.15.37:0/484483940' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi037.lykblh/key"}]: dispatch 2024-09-20T22:21:03.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:03 smithi194 ceph-mon[25957]: from='mgr.? 172.21.15.37:0/484483940' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-20T22:21:03.815 INFO:teuthology.orchestra.run.smithi037.stdout: 5 94529 12.53 MB/sec execute 81 sec latency 31.228 ms 2024-09-20T22:21:04.412 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:04 smithi037 ceph-mon[23718]: pgmap v49: 97 pgs: 97 active+clean; 323 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 6.0 MiB/s wr, 611 op/s 2024-09-20T22:21:04.412 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:04 smithi037 ceph-mon[23718]: mgrmap e28: smithi194.bkcglt(active, since 93s), standbys: smithi037.lykblh 2024-09-20T22:21:04.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:04 smithi194 ceph-mon[25957]: pgmap v49: 97 pgs: 97 active+clean; 323 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 6.0 MiB/s wr, 611 op/s 2024-09-20T22:21:04.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:04 smithi194 ceph-mon[25957]: mgrmap e28: smithi194.bkcglt(active, since 93s), standbys: smithi037.lykblh 2024-09-20T22:21:04.815 INFO:teuthology.orchestra.run.smithi037.stdout: 5 94868 12.45 MB/sec execute 82 sec latency 53.607 ms 2024-09-20T22:21:05.456 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:05 smithi037 ceph-mon[23718]: mgrmap e29: smithi194.bkcglt(active, since 94s), standbys: smithi037.lykblh 2024-09-20T22:21:05.456 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:05 smithi037 ceph-mon[23718]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:21:05.456 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:05 smithi037 ceph-mon[23718]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:21:05.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:05 smithi194 ceph-mon[25957]: mgrmap e29: smithi194.bkcglt(active, since 94s), standbys: smithi037.lykblh 2024-09-20T22:21:05.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:05 smithi194 ceph-mon[25957]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:21:05.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:05 smithi194 ceph-mon[25957]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:21:05.815 INFO:teuthology.orchestra.run.smithi037.stdout: 5 95190 12.49 MB/sec execute 83 sec latency 69.524 ms 2024-09-20T22:21:06.816 INFO:teuthology.orchestra.run.smithi037.stdout: 5 95567 12.51 MB/sec execute 84 sec latency 89.134 ms 2024-09-20T22:21:07.232 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:06 smithi037 ceph-mon[23718]: pgmap v50: 97 pgs: 97 active+clean; 323 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.1 MiB/s wr, 383 op/s 2024-09-20T22:21:07.233 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:06 smithi037 ceph-mon[23718]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:21:07.233 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:06 smithi037 ceph-mon[23718]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:21:07.346 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:06 smithi194 ceph-mon[25957]: pgmap v50: 97 pgs: 97 active+clean; 323 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.1 MiB/s wr, 383 op/s 2024-09-20T22:21:07.346 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:06 smithi194 ceph-mon[25957]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:21:07.346 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:06 smithi194 ceph-mon[25957]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:21:07.815 INFO:teuthology.orchestra.run.smithi037.stdout: 5 95928 12.55 MB/sec execute 85 sec latency 43.929 ms 2024-09-20T22:21:08.816 INFO:teuthology.orchestra.run.smithi037.stdout: 5 96195 12.48 MB/sec execute 86 sec latency 49.933 ms 2024-09-20T22:21:09.032 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:08 smithi037 ceph-mon[23718]: pgmap v51: 97 pgs: 97 active+clean; 326 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.5 MiB/s wr, 398 op/s 2024-09-20T22:21:09.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:08 smithi194 ceph-mon[25957]: pgmap v51: 97 pgs: 97 active+clean; 326 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.5 MiB/s wr, 398 op/s 2024-09-20T22:21:09.816 INFO:teuthology.orchestra.run.smithi037.stdout: 5 96395 12.37 MB/sec execute 87 sec latency 69.009 ms 2024-09-20T22:21:10.816 INFO:teuthology.orchestra.run.smithi037.stdout: 5 96809 12.30 MB/sec execute 88 sec latency 37.538 ms 2024-09-20T22:21:11.103 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:10 smithi037 ceph-mon[23718]: pgmap v52: 97 pgs: 97 active+clean; 333 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 6.5 MiB/s wr, 594 op/s 2024-09-20T22:21:11.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:10 smithi194 ceph-mon[25957]: pgmap v52: 97 pgs: 97 active+clean; 333 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 6.5 MiB/s wr, 594 op/s 2024-09-20T22:21:11.816 INFO:teuthology.orchestra.run.smithi037.stdout: 5 97184 12.25 MB/sec execute 89 sec latency 44.656 ms 2024-09-20T22:21:12.824 INFO:teuthology.orchestra.run.smithi037.stdout: 5 97567 12.22 MB/sec execute 90 sec latency 64.912 ms 2024-09-20T22:21:13.057 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:12 smithi037 ceph-mon[23718]: pgmap v53: 97 pgs: 97 active+clean; 333 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.9 MiB/s wr, 393 op/s 2024-09-20T22:21:13.057 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:12 smithi037 ceph-mon[23718]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:21:13.057 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:12 smithi037 ceph-mon[23718]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:21:13.057 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:12 smithi037 ceph-mon[23718]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:21:13.057 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:12 smithi037 ceph-mon[23718]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:21:13.058 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:12 smithi037 ceph-mon[23718]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:21:13.058 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:12 smithi037 ceph-mon[23718]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:21:13.058 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:12 smithi037 ceph-mon[23718]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:21:13.058 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:12 smithi037 ceph-mon[23718]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "mgr fail", "who": "smithi194.bkcglt"}]: dispatch 2024-09-20T22:21:13.058 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:12 smithi037 ceph-mon[23718]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "mgr fail", "who": "smithi194.bkcglt"}]: dispatch 2024-09-20T22:21:13.058 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:12 smithi037 ceph-mon[23718]: osdmap e71: 8 total, 8 up, 8 in 2024-09-20T22:21:13.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:12 smithi194 ceph-mon[25957]: pgmap v53: 97 pgs: 97 active+clean; 333 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.9 MiB/s wr, 393 op/s 2024-09-20T22:21:13.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:12 smithi194 ceph-mon[25957]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:21:13.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:12 smithi194 ceph-mon[25957]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:21:13.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:12 smithi194 ceph-mon[25957]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:21:13.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:12 smithi194 ceph-mon[25957]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:21:13.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:12 smithi194 ceph-mon[25957]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:21:13.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:12 smithi194 ceph-mon[25957]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' 2024-09-20T22:21:13.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:12 smithi194 ceph-mon[25957]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:21:13.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:12 smithi194 ceph-mon[25957]: from='mgr.24437 172.21.15.194:0/4284873836' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "mgr fail", "who": "smithi194.bkcglt"}]: dispatch 2024-09-20T22:21:13.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:12 smithi194 ceph-mon[25957]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "mgr fail", "who": "smithi194.bkcglt"}]: dispatch 2024-09-20T22:21:13.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:12 smithi194 ceph-mon[25957]: osdmap e71: 8 total, 8 up, 8 in 2024-09-20T22:21:13.824 INFO:teuthology.orchestra.run.smithi037.stdout: 5 97938 12.15 MB/sec execute 91 sec latency 53.944 ms 2024-09-20T22:21:14.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:14 smithi194 ceph-mon[25957]: Upgrade: Need to upgrade myself (mgr.smithi194.bkcglt) 2024-09-20T22:21:14.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:14 smithi194 ceph-mon[25957]: Upgrade: Need to upgrade myself (mgr.smithi194.bkcglt) 2024-09-20T22:21:14.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:14 smithi194 ceph-mon[25957]: Failing over to other MGR 2024-09-20T22:21:14.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:14 smithi194 ceph-mon[25957]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' cmd='[{"prefix": "mgr fail", "who": "smithi194.bkcglt"}]': finished 2024-09-20T22:21:14.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:14 smithi194 ceph-mon[25957]: mgrmap e30: smithi037.lykblh(active, starting, since 0.855804s) 2024-09-20T22:21:14.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:14 smithi194 ceph-mon[25957]: from='mgr.14772 172.21.15.37:0/484483940' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mon metadata", "id": "smithi037"}]: dispatch 2024-09-20T22:21:14.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:14 smithi194 ceph-mon[25957]: from='mgr.14772 172.21.15.37:0/484483940' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mon metadata", "id": "smithi194"}]: dispatch 2024-09-20T22:21:14.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:14 smithi194 ceph-mon[25957]: from='mgr.14772 172.21.15.37:0/484483940' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi037.sdtedi"}]: dispatch 2024-09-20T22:21:14.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:14 smithi194 ceph-mon[25957]: from='mgr.14772 172.21.15.37:0/484483940' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi194.lkroso"}]: dispatch 2024-09-20T22:21:14.132 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:13 smithi037 ceph-mon[23718]: Upgrade: Need to upgrade myself (mgr.smithi194.bkcglt) 2024-09-20T22:21:14.132 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:13 smithi037 ceph-mon[23718]: Upgrade: Need to upgrade myself (mgr.smithi194.bkcglt) 2024-09-20T22:21:14.132 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:13 smithi037 ceph-mon[23718]: Failing over to other MGR 2024-09-20T22:21:14.133 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:13 smithi037 ceph-mon[23718]: from='mgr.24437 ' entity='mgr.smithi194.bkcglt' cmd='[{"prefix": "mgr fail", "who": "smithi194.bkcglt"}]': finished 2024-09-20T22:21:14.133 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:13 smithi037 ceph-mon[23718]: mgrmap e30: smithi037.lykblh(active, starting, since 0.855804s) 2024-09-20T22:21:14.133 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:13 smithi037 ceph-mon[23718]: from='mgr.14772 172.21.15.37:0/484483940' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mon metadata", "id": "smithi037"}]: dispatch 2024-09-20T22:21:14.133 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:13 smithi037 ceph-mon[23718]: from='mgr.14772 172.21.15.37:0/484483940' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mon metadata", "id": "smithi194"}]: dispatch 2024-09-20T22:21:14.133 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:13 smithi037 ceph-mon[23718]: from='mgr.14772 172.21.15.37:0/484483940' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi037.sdtedi"}]: dispatch 2024-09-20T22:21:14.133 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:13 smithi037 ceph-mon[23718]: from='mgr.14772 172.21.15.37:0/484483940' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi194.lkroso"}]: dispatch 2024-09-20T22:21:14.824 INFO:teuthology.orchestra.run.smithi037.stdout: 5 98275 12.11 MB/sec execute 92 sec latency 57.700 ms 2024-09-20T22:21:15.824 INFO:teuthology.orchestra.run.smithi037.stdout: 5 98578 12.09 MB/sec execute 93 sec latency 40.656 ms 2024-09-20T22:21:16.824 INFO:teuthology.orchestra.run.smithi037.stdout: 5 98878 12.11 MB/sec execute 94 sec latency 65.747 ms 2024-09-20T22:21:17.824 INFO:teuthology.orchestra.run.smithi037.stdout: 5 99147 12.10 MB/sec execute 95 sec latency 52.094 ms 2024-09-20T22:21:18.824 INFO:teuthology.orchestra.run.smithi037.stdout: 5 99347 12.05 MB/sec execute 96 sec latency 78.513 ms 2024-09-20T22:21:19.045 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:18 smithi037 ceph-mon[23718]: Standby manager daemon smithi194.bkcglt started 2024-09-20T22:21:19.045 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:18 smithi037 ceph-mon[23718]: from='mgr.? 172.21.15.194:0/1809693680' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi194.bkcglt/crt"}]: dispatch 2024-09-20T22:21:19.046 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:18 smithi037 ceph-mon[23718]: from='mgr.? 172.21.15.194:0/1809693680' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-20T22:21:19.046 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:18 smithi037 ceph-mon[23718]: from='mgr.? 172.21.15.194:0/1809693680' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi194.bkcglt/key"}]: dispatch 2024-09-20T22:21:19.046 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:18 smithi037 ceph-mon[23718]: from='mgr.? 172.21.15.194:0/1809693680' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-20T22:21:19.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:18 smithi194 ceph-mon[25957]: Standby manager daemon smithi194.bkcglt started 2024-09-20T22:21:19.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:18 smithi194 ceph-mon[25957]: from='mgr.? 172.21.15.194:0/1809693680' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi194.bkcglt/crt"}]: dispatch 2024-09-20T22:21:19.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:18 smithi194 ceph-mon[25957]: from='mgr.? 172.21.15.194:0/1809693680' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-20T22:21:19.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:18 smithi194 ceph-mon[25957]: from='mgr.? 172.21.15.194:0/1809693680' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi194.bkcglt/key"}]: dispatch 2024-09-20T22:21:19.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:18 smithi194 ceph-mon[25957]: from='mgr.? 172.21.15.194:0/1809693680' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-20T22:21:19.824 INFO:teuthology.orchestra.run.smithi037.stdout: 5 99593 11.99 MB/sec execute 97 sec latency 77.627 ms 2024-09-20T22:21:20.114 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:19 smithi037 ceph-mon[23718]: mgrmap e31: smithi037.lykblh(active, starting, since 6s), standbys: smithi194.bkcglt 2024-09-20T22:21:20.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:19 smithi194 ceph-mon[25957]: mgrmap e31: smithi037.lykblh(active, starting, since 6s), standbys: smithi194.bkcglt 2024-09-20T22:21:20.824 INFO:teuthology.orchestra.run.smithi037.stdout: 5 99797 11.93 MB/sec execute 98 sec latency 85.052 ms 2024-09-20T22:21:21.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:20 smithi037 ceph-mon[23718]: Active manager daemon smithi037.lykblh restarted 2024-09-20T22:21:21.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:20 smithi037 ceph-mon[23718]: Activating manager daemon smithi037.lykblh 2024-09-20T22:21:21.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:20 smithi037 ceph-mon[23718]: osdmap e72: 8 total, 8 up, 8 in 2024-09-20T22:21:21.325 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:20 smithi194 ceph-mon[25957]: Active manager daemon smithi037.lykblh restarted 2024-09-20T22:21:21.325 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:20 smithi194 ceph-mon[25957]: Activating manager daemon smithi037.lykblh 2024-09-20T22:21:21.326 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:20 smithi194 ceph-mon[25957]: osdmap e72: 8 total, 8 up, 8 in 2024-09-20T22:21:21.824 INFO:teuthology.orchestra.run.smithi037.stdout: 5 100011 11.84 MB/sec execute 99 sec latency 57.646 ms 2024-09-20T22:21:21.943 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:21 smithi037 ceph-mon[23718]: mgrmap e32: smithi037.lykblh(active, starting, since 0.876764s), standbys: smithi194.bkcglt 2024-09-20T22:21:21.944 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:21 smithi037 ceph-mon[23718]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mon metadata", "id": "smithi037"}]: dispatch 2024-09-20T22:21:21.944 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:21 smithi037 ceph-mon[23718]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mon metadata", "id": "smithi194"}]: dispatch 2024-09-20T22:21:21.944 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:21 smithi037 ceph-mon[23718]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi037.sdtedi"}]: dispatch 2024-09-20T22:21:21.944 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:21 smithi037 ceph-mon[23718]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi194.lkroso"}]: dispatch 2024-09-20T22:21:21.944 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:21 smithi037 ceph-mon[23718]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mgr metadata", "who": "smithi037.lykblh", "id": "smithi037.lykblh"}]: dispatch 2024-09-20T22:21:21.944 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:21 smithi037 ceph-mon[23718]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mgr metadata", "who": "smithi194.bkcglt", "id": "smithi194.bkcglt"}]: dispatch 2024-09-20T22:21:21.944 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:21 smithi037 ceph-mon[23718]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-20T22:21:21.944 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:21 smithi037 ceph-mon[23718]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-20T22:21:21.944 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:21 smithi037 ceph-mon[23718]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-20T22:21:21.944 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:21 smithi037 ceph-mon[23718]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-20T22:21:21.944 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:21 smithi037 ceph-mon[23718]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-20T22:21:21.944 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:21 smithi037 ceph-mon[23718]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-20T22:21:21.944 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:21 smithi037 ceph-mon[23718]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-20T22:21:21.945 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:21 smithi037 ceph-mon[23718]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-20T22:21:21.945 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:21 smithi037 ceph-mon[23718]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-20T22:21:21.950 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:21 smithi037 ceph-mon[23718]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-20T22:21:21.950 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:21 smithi037 ceph-mon[23718]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-20T22:21:21.950 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:21 smithi037 ceph-mon[23718]: Manager daemon smithi037.lykblh is now available 2024-09-20T22:21:21.950 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:21 smithi037 ceph-mon[23718]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:21:21.951 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:21 smithi037 ceph-mon[23718]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:21:21.951 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:21 smithi037 ceph-mon[23718]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi037.lykblh/mirror_snapshot_schedule"}]: dispatch 2024-09-20T22:21:21.951 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:21 smithi037 ceph-mon[23718]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi037.lykblh/trash_purge_schedule"}]: dispatch 2024-09-20T22:21:22.340 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:21 smithi194 ceph-mon[25957]: mgrmap e32: smithi037.lykblh(active, starting, since 0.876764s), standbys: smithi194.bkcglt 2024-09-20T22:21:22.340 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:21 smithi194 ceph-mon[25957]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mon metadata", "id": "smithi037"}]: dispatch 2024-09-20T22:21:22.340 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:21 smithi194 ceph-mon[25957]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mon metadata", "id": "smithi194"}]: dispatch 2024-09-20T22:21:22.340 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:21 smithi194 ceph-mon[25957]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi037.sdtedi"}]: dispatch 2024-09-20T22:21:22.340 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:21 smithi194 ceph-mon[25957]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi194.lkroso"}]: dispatch 2024-09-20T22:21:22.340 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:21 smithi194 ceph-mon[25957]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mgr metadata", "who": "smithi037.lykblh", "id": "smithi037.lykblh"}]: dispatch 2024-09-20T22:21:22.340 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:21 smithi194 ceph-mon[25957]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mgr metadata", "who": "smithi194.bkcglt", "id": "smithi194.bkcglt"}]: dispatch 2024-09-20T22:21:22.340 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:21 smithi194 ceph-mon[25957]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-20T22:21:22.341 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:21 smithi194 ceph-mon[25957]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-20T22:21:22.341 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:21 smithi194 ceph-mon[25957]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-20T22:21:22.341 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:21 smithi194 ceph-mon[25957]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-20T22:21:22.341 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:21 smithi194 ceph-mon[25957]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-20T22:21:22.341 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:21 smithi194 ceph-mon[25957]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-20T22:21:22.341 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:21 smithi194 ceph-mon[25957]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-20T22:21:22.341 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:21 smithi194 ceph-mon[25957]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-20T22:21:22.341 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:21 smithi194 ceph-mon[25957]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-20T22:21:22.341 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:21 smithi194 ceph-mon[25957]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-20T22:21:22.341 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:21 smithi194 ceph-mon[25957]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-20T22:21:22.341 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:21 smithi194 ceph-mon[25957]: Manager daemon smithi037.lykblh is now available 2024-09-20T22:21:22.341 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:21 smithi194 ceph-mon[25957]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:21:22.341 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:21 smithi194 ceph-mon[25957]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:21:22.341 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:21 smithi194 ceph-mon[25957]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi037.lykblh/mirror_snapshot_schedule"}]: dispatch 2024-09-20T22:21:22.341 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:21 smithi194 ceph-mon[25957]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi037.lykblh/trash_purge_schedule"}]: dispatch 2024-09-20T22:21:22.824 INFO:teuthology.orchestra.run.smithi037.stdout: 5 100251 11.77 MB/sec execute 100 sec latency 94.479 ms 2024-09-20T22:21:23.219 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:22 smithi037 ceph-mon[23718]: mgrmap e33: smithi037.lykblh(active, since 1.89378s), standbys: smithi194.bkcglt 2024-09-20T22:21:23.220 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:22 smithi037 ceph-mon[23718]: pgmap v3: 97 pgs: 97 active+clean; 333 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-20T22:21:23.255 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:22 smithi194 ceph-mon[25957]: mgrmap e33: smithi037.lykblh(active, since 1.89378s), standbys: smithi194.bkcglt 2024-09-20T22:21:23.255 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:22 smithi194 ceph-mon[25957]: pgmap v3: 97 pgs: 97 active+clean; 333 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-20T22:21:23.824 INFO:teuthology.orchestra.run.smithi037.stdout: 5 100513 11.69 MB/sec execute 101 sec latency 49.755 ms 2024-09-20T22:21:24.000 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:23 smithi037 ceph-mon[23718]: pgmap v4: 97 pgs: 97 active+clean; 333 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-20T22:21:24.000 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:23 smithi037 ceph-mon[23718]: mgrmap e34: smithi037.lykblh(active, since 2s), standbys: smithi194.bkcglt 2024-09-20T22:21:24.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:23 smithi194 ceph-mon[25957]: pgmap v4: 97 pgs: 97 active+clean; 333 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-20T22:21:24.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:23 smithi194 ceph-mon[25957]: mgrmap e34: smithi037.lykblh(active, since 2s), standbys: smithi194.bkcglt 2024-09-20T22:21:24.824 INFO:teuthology.orchestra.run.smithi037.stdout: 5 100938 11.66 MB/sec execute 102 sec latency 77.353 ms 2024-09-20T22:21:25.004 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:24 smithi037 ceph-mon[23718]: [20/Sep/2024:22:21:23] ENGINE Bus STARTING 2024-09-20T22:21:25.005 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:24 smithi037 ceph-mon[23718]: [20/Sep/2024:22:21:23] ENGINE Serving on https://172.21.15.37:7150 2024-09-20T22:21:25.005 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:24 smithi037 ceph-mon[23718]: [20/Sep/2024:22:21:23] ENGINE Error in HTTPServer.serve 2024-09-20T22:21:25.005 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: Traceback (most recent call last): 2024-09-20T22:21:25.005 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-20T22:21:25.005 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: self._connections.run(self.expiration_interval) 2024-09-20T22:21:25.005 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-20T22:21:25.005 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: self._run(expiration_interval) 2024-09-20T22:21:25.005 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-20T22:21:25.005 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-20T22:21:25.005 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-20T22:21:25.005 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-20T22:21:25.005 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-20T22:21:25.006 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: s = self.context.wrap_socket( 2024-09-20T22:21:25.006 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-20T22:21:25.006 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: return self.sslsocket_class._create( 2024-09-20T22:21:25.006 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-20T22:21:25.006 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: self.do_handshake() 2024-09-20T22:21:25.006 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-20T22:21:25.006 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: self._sslobj.do_handshake() 2024-09-20T22:21:25.006 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-20T22:21:25.006 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:24 smithi037 ceph-mon[23718]: [20/Sep/2024:22:21:23] ENGINE Serving on http://172.21.15.37:8765 2024-09-20T22:21:25.006 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:24 smithi037 ceph-mon[23718]: [20/Sep/2024:22:21:23] ENGINE Bus STARTED 2024-09-20T22:21:25.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:24 smithi194 ceph-mon[25957]: [20/Sep/2024:22:21:23] ENGINE Bus STARTING 2024-09-20T22:21:25.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:24 smithi194 ceph-mon[25957]: [20/Sep/2024:22:21:23] ENGINE Serving on https://172.21.15.37:7150 2024-09-20T22:21:25.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:24 smithi194 ceph-mon[25957]: [20/Sep/2024:22:21:23] ENGINE Error in HTTPServer.serve 2024-09-20T22:21:25.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: Traceback (most recent call last): 2024-09-20T22:21:25.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-20T22:21:25.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: self._connections.run(self.expiration_interval) 2024-09-20T22:21:25.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-20T22:21:25.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: self._run(expiration_interval) 2024-09-20T22:21:25.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-20T22:21:25.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-20T22:21:25.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-20T22:21:25.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-20T22:21:25.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-20T22:21:25.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: s = self.context.wrap_socket( 2024-09-20T22:21:25.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-20T22:21:25.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: return self.sslsocket_class._create( 2024-09-20T22:21:25.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-20T22:21:25.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: self.do_handshake() 2024-09-20T22:21:25.096 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-20T22:21:25.096 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: self._sslobj.do_handshake() 2024-09-20T22:21:25.096 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-20T22:21:25.096 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:24 smithi194 ceph-mon[25957]: [20/Sep/2024:22:21:23] ENGINE Serving on http://172.21.15.37:8765 2024-09-20T22:21:25.096 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:24 smithi194 ceph-mon[25957]: [20/Sep/2024:22:21:23] ENGINE Bus STARTED 2024-09-20T22:21:25.824 INFO:teuthology.orchestra.run.smithi037.stdout: 5 101353 11.63 MB/sec execute 103 sec latency 43.862 ms 2024-09-20T22:21:26.012 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:25 smithi037 ceph-mon[23718]: pgmap v5: 97 pgs: 97 active+clean; 333 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-20T22:21:26.012 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:25 smithi037 ceph-mon[23718]: mgrmap e35: smithi037.lykblh(active, since 4s), standbys: smithi194.bkcglt 2024-09-20T22:21:26.012 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:25 smithi037 ceph-mon[23718]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' 2024-09-20T22:21:26.012 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:25 smithi037 ceph-mon[23718]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' 2024-09-20T22:21:26.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:25 smithi194 ceph-mon[25957]: pgmap v5: 97 pgs: 97 active+clean; 333 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-20T22:21:26.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:25 smithi194 ceph-mon[25957]: mgrmap e35: smithi037.lykblh(active, since 4s), standbys: smithi194.bkcglt 2024-09-20T22:21:26.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:25 smithi194 ceph-mon[25957]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' 2024-09-20T22:21:26.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:25 smithi194 ceph-mon[25957]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' 2024-09-20T22:21:26.824 INFO:teuthology.orchestra.run.smithi037.stdout: 5 101727 11.62 MB/sec execute 104 sec latency 41.954 ms 2024-09-20T22:21:27.288 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:27 smithi037 ceph-mon[23718]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' 2024-09-20T22:21:27.288 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:27 smithi037 ceph-mon[23718]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' 2024-09-20T22:21:27.324 INFO:teuthology.orchestra.run.smithi037.stdout:true 2024-09-20T22:21:27.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:27 smithi194 ceph-mon[25957]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' 2024-09-20T22:21:27.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:27 smithi194 ceph-mon[25957]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' 2024-09-20T22:21:27.824 INFO:teuthology.orchestra.run.smithi037.stdout: 5 102086 11.61 MB/sec execute 105 sec latency 56.266 ms 2024-09-20T22:21:28.331 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:28 smithi037 ceph-mon[23718]: pgmap v6: 97 pgs: 97 active+clean; 333 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-20T22:21:28.331 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:28 smithi037 ceph-mon[23718]: from='client.14808 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:21:28.365 INFO:teuthology.orchestra.run.smithi037.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T22:21:28.365 INFO:teuthology.orchestra.run.smithi037.stdout:alertmanager.smithi037 smithi037 *:9093,9094 running (86s) 23s ago 10m 27.9M - 0.25.0 c8568f914cd2 d1f99b3cd707 2024-09-20T22:21:28.365 INFO:teuthology.orchestra.run.smithi037.stdout:crash.smithi037 smithi037 running (10m) 23s ago 10m 6400k - 17.2.0 e1d6a67b021e 88301022bdc8 2024-09-20T22:21:28.365 INFO:teuthology.orchestra.run.smithi037.stdout:crash.smithi194 smithi194 running (9m) 2s ago 9m 6405k - 17.2.0 e1d6a67b021e b553601d7ca5 2024-09-20T22:21:28.365 INFO:teuthology.orchestra.run.smithi037.stdout:grafana.smithi037 smithi037 *:3000 running (77s) 23s ago 10m 45.7M - dad864ee21e9 2306f3c2da19 2024-09-20T22:21:28.365 INFO:teuthology.orchestra.run.smithi037.stdout:mds.foofs.smithi037.sdtedi smithi037 running (4m) 23s ago 4m 36.5M - 17.2.0 e1d6a67b021e 0a7898d25b8d 2024-09-20T22:21:28.366 INFO:teuthology.orchestra.run.smithi037.stdout:mds.foofs.smithi194.lkroso smithi194 running (4m) 2s ago 4m 15.1M - 17.2.0 e1d6a67b021e 17736de30008 2024-09-20T22:21:28.366 INFO:teuthology.orchestra.run.smithi037.stdout:mgr.smithi037.lykblh smithi037 *:8443,9283,8765 running (32s) 23s ago 11m 108M - 19.1.1-367-g9e2ac70b cc9251b744a3 181e46fd9ba6 2024-09-20T22:21:28.366 INFO:teuthology.orchestra.run.smithi037.stdout:mgr.smithi194.bkcglt smithi194 *:8443,9283 running (2m) 2s ago 9m 437M - 19.1.1-367-g9e2ac70b cc9251b744a3 e0af2512175a 2024-09-20T22:21:28.366 INFO:teuthology.orchestra.run.smithi037.stdout:mon.smithi037 smithi037 running (11m) 23s ago 11m 52.0M 2048M 17.2.0 e1d6a67b021e 71f591122786 2024-09-20T22:21:28.366 INFO:teuthology.orchestra.run.smithi037.stdout:mon.smithi194 smithi194 running (9m) 2s ago 9m 44.9M 2048M 17.2.0 e1d6a67b021e 0b8814ae6c66 2024-09-20T22:21:28.366 INFO:teuthology.orchestra.run.smithi037.stdout:nfs.foo.0.0.smithi037.spabdc smithi037 *:2049 running (3m) 23s ago 3m 258M - 4.0 e1d6a67b021e 858ff9d558a2 2024-09-20T22:21:28.366 INFO:teuthology.orchestra.run.smithi037.stdout:nfs.foo.1.0.smithi194.qzjyej smithi194 *:2049 running (3m) 2s ago 3m 85.6M - 4.0 e1d6a67b021e c07fe0eb7c75 2024-09-20T22:21:28.366 INFO:teuthology.orchestra.run.smithi037.stdout:node-exporter.smithi037 smithi037 *:9100 running (69s) 23s ago 9m 7931k - 1.5.0 0da6a335fe13 433e6aaf6137 2024-09-20T22:21:28.366 INFO:teuthology.orchestra.run.smithi037.stdout:node-exporter.smithi194 smithi194 *:9100 running (41s) 2s ago 9m 7776k - 1.5.0 0da6a335fe13 1d30bb810ad1 2024-09-20T22:21:28.366 INFO:teuthology.orchestra.run.smithi037.stdout:osd.0 smithi037 running (8m) 23s ago 8m 369M 1937M 17.2.0 e1d6a67b021e e54e84fd5030 2024-09-20T22:21:28.366 INFO:teuthology.orchestra.run.smithi037.stdout:osd.1 smithi037 running (8m) 23s ago 8m 275M 1937M 17.2.0 e1d6a67b021e 09647c133032 2024-09-20T22:21:28.366 INFO:teuthology.orchestra.run.smithi037.stdout:osd.2 smithi037 running (7m) 23s ago 7m 246M 1937M 17.2.0 e1d6a67b021e 6c15b1175256 2024-09-20T22:21:28.366 INFO:teuthology.orchestra.run.smithi037.stdout:osd.3 smithi037 running (7m) 23s ago 7m 339M 1937M 17.2.0 e1d6a67b021e 3a22952c6475 2024-09-20T22:21:28.367 INFO:teuthology.orchestra.run.smithi037.stdout:osd.4 smithi194 running (6m) 2s ago 6m 357M 2704M 17.2.0 e1d6a67b021e fa107be8915c 2024-09-20T22:21:28.367 INFO:teuthology.orchestra.run.smithi037.stdout:osd.5 smithi194 running (6m) 2s ago 6m 396M 2704M 17.2.0 e1d6a67b021e 290b21cbd131 2024-09-20T22:21:28.367 INFO:teuthology.orchestra.run.smithi037.stdout:osd.6 smithi194 running (6m) 2s ago 6m 332M 2704M 17.2.0 e1d6a67b021e 27f0720639ed 2024-09-20T22:21:28.367 INFO:teuthology.orchestra.run.smithi037.stdout:osd.7 smithi194 running (5m) 2s ago 5m 477M 2704M 17.2.0 e1d6a67b021e 62d163f4fd92 2024-09-20T22:21:28.367 INFO:teuthology.orchestra.run.smithi037.stdout:prometheus.smithi037 smithi037 *:9095 running (47s) 23s ago 9m 44.0M - 2.43.0 a07b618ecd1d 929a584b912f 2024-09-20T22:21:28.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:28 smithi194 ceph-mon[25957]: pgmap v6: 97 pgs: 97 active+clean; 333 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-20T22:21:28.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:28 smithi194 ceph-mon[25957]: from='client.14808 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:21:28.824 INFO:teuthology.orchestra.run.smithi037.stdout: 5 102442 11.64 MB/sec execute 106 sec latency 57.345 ms 2024-09-20T22:21:28.992 INFO:teuthology.orchestra.run.smithi037.stdout:{ 2024-09-20T22:21:28.992 INFO:teuthology.orchestra.run.smithi037.stdout: "mon": { 2024-09-20T22:21:28.992 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T22:21:28.992 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:21:28.992 INFO:teuthology.orchestra.run.smithi037.stdout: "mgr": { 2024-09-20T22:21:28.993 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 2 2024-09-20T22:21:28.993 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:21:28.993 INFO:teuthology.orchestra.run.smithi037.stdout: "osd": { 2024-09-20T22:21:28.993 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-20T22:21:28.993 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:21:28.993 INFO:teuthology.orchestra.run.smithi037.stdout: "mds": { 2024-09-20T22:21:28.993 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T22:21:28.993 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:21:28.993 INFO:teuthology.orchestra.run.smithi037.stdout: "overall": { 2024-09-20T22:21:28.993 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-09-20T22:21:28.993 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 2 2024-09-20T22:21:28.993 INFO:teuthology.orchestra.run.smithi037.stdout: } 2024-09-20T22:21:28.993 INFO:teuthology.orchestra.run.smithi037.stdout:} 2024-09-20T22:21:29.381 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:29 smithi037 ceph-mon[23718]: from='client.14812 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:21:29.381 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:29 smithi037 ceph-mon[23718]: from='client.14816 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:21:29.381 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:29 smithi037 ceph-mon[23718]: from='client.? 172.21.15.37:0/2613984664' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:21:29.561 INFO:teuthology.orchestra.run.smithi037.stdout:{ 2024-09-20T22:21:29.561 INFO:teuthology.orchestra.run.smithi037.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5", 2024-09-20T22:21:29.561 INFO:teuthology.orchestra.run.smithi037.stdout: "in_progress": true, 2024-09-20T22:21:29.561 INFO:teuthology.orchestra.run.smithi037.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-20T22:21:29.561 INFO:teuthology.orchestra.run.smithi037.stdout: "services_complete": [ 2024-09-20T22:21:29.561 INFO:teuthology.orchestra.run.smithi037.stdout: "mgr" 2024-09-20T22:21:29.561 INFO:teuthology.orchestra.run.smithi037.stdout: ], 2024-09-20T22:21:29.561 INFO:teuthology.orchestra.run.smithi037.stdout: "progress": "2/23 daemons upgraded", 2024-09-20T22:21:29.561 INFO:teuthology.orchestra.run.smithi037.stdout: "message": "", 2024-09-20T22:21:29.561 INFO:teuthology.orchestra.run.smithi037.stdout: "is_paused": false 2024-09-20T22:21:29.561 INFO:teuthology.orchestra.run.smithi037.stdout:} 2024-09-20T22:21:29.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:29 smithi194 ceph-mon[25957]: from='client.14812 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:21:29.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:29 smithi194 ceph-mon[25957]: from='client.14816 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:21:29.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:29 smithi194 ceph-mon[25957]: from='client.? 172.21.15.37:0/2613984664' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:21:29.824 INFO:teuthology.orchestra.run.smithi037.stdout: 5 102786 11.67 MB/sec execute 107 sec latency 54.545 ms 2024-09-20T22:21:30.215 INFO:teuthology.orchestra.run.smithi037.stdout:HEALTH_OK 2024-09-20T22:21:30.398 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:30 smithi037 ceph-mon[23718]: pgmap v7: 97 pgs: 97 active+clean; 327 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 36 KiB/s rd, 3.1 MiB/s wr, 390 op/s 2024-09-20T22:21:30.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:30 smithi194 ceph-mon[25957]: pgmap v7: 97 pgs: 97 active+clean; 327 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 36 KiB/s rd, 3.1 MiB/s wr, 390 op/s 2024-09-20T22:21:30.825 INFO:teuthology.orchestra.run.smithi037.stdout: 5 103089 11.66 MB/sec execute 108 sec latency 87.591 ms 2024-09-20T22:21:31.436 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:31 smithi037 ceph-mon[23718]: from='client.14824 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:21:31.436 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:31 smithi037 ceph-mon[23718]: from='client.? 172.21.15.37:0/3250331221' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T22:21:31.436 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:31 smithi037 ceph-mon[23718]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' 2024-09-20T22:21:31.436 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:31 smithi037 ceph-mon[23718]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' 2024-09-20T22:21:31.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:31 smithi194 ceph-mon[25957]: from='client.14824 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:21:31.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:31 smithi194 ceph-mon[25957]: from='client.? 172.21.15.37:0/3250331221' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T22:21:31.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:31 smithi194 ceph-mon[25957]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' 2024-09-20T22:21:31.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:31 smithi194 ceph-mon[25957]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' 2024-09-20T22:21:31.825 INFO:teuthology.orchestra.run.smithi037.stdout: 5 103367 11.60 MB/sec execute 109 sec latency 47.009 ms 2024-09-20T22:21:32.442 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:32 smithi037 ceph-mon[23718]: pgmap v8: 97 pgs: 97 active+clean; 327 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 28 KiB/s rd, 2.4 MiB/s wr, 302 op/s 2024-09-20T22:21:32.443 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:32 smithi037 ceph-mon[23718]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' 2024-09-20T22:21:32.443 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:32 smithi037 ceph-mon[23718]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' 2024-09-20T22:21:32.824 INFO:teuthology.orchestra.run.smithi037.stdout: 5 103614 11.53 MB/sec execute 110 sec latency 48.626 ms 2024-09-20T22:21:32.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:32 smithi194 ceph-mon[25957]: pgmap v8: 97 pgs: 97 active+clean; 327 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 28 KiB/s rd, 2.4 MiB/s wr, 302 op/s 2024-09-20T22:21:32.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:32 smithi194 ceph-mon[25957]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' 2024-09-20T22:21:32.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:32 smithi194 ceph-mon[25957]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' 2024-09-20T22:21:33.824 INFO:teuthology.orchestra.run.smithi037.stdout: 5 104008 11.49 MB/sec execute 111 sec latency 91.712 ms 2024-09-20T22:21:34.474 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:34 smithi037 ceph-mon[23718]: pgmap v9: 97 pgs: 97 active+clean; 339 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 4.5 MiB/s wr, 451 op/s 2024-09-20T22:21:34.825 INFO:teuthology.orchestra.run.smithi037.stdout: 5 104678 11.52 MB/sec execute 112 sec latency 32.315 ms 2024-09-20T22:21:34.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:34 smithi194 ceph-mon[25957]: pgmap v9: 97 pgs: 97 active+clean; 339 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 4.5 MiB/s wr, 451 op/s 2024-09-20T22:21:35.742 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:35 smithi037 ceph-mon[23718]: pgmap v10: 97 pgs: 97 active+clean; 339 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 4.1 MiB/s wr, 408 op/s 2024-09-20T22:21:35.824 INFO:teuthology.orchestra.run.smithi037.stdout: 5 105376 11.58 MB/sec execute 113 sec latency 42.532 ms 2024-09-20T22:21:35.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:35 smithi194 ceph-mon[25957]: pgmap v10: 97 pgs: 97 active+clean; 339 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 4.1 MiB/s wr, 408 op/s 2024-09-20T22:21:36.746 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:36 smithi037 ceph-mon[23718]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:21:36.825 INFO:teuthology.orchestra.run.smithi037.stdout: 5 106286 11.77 MB/sec execute 114 sec latency 18.193 ms 2024-09-20T22:21:36.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:36 smithi194 ceph-mon[25957]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:21:37.754 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:37 smithi037 ceph-mon[23718]: pgmap v11: 97 pgs: 97 active+clean; 339 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 4.1 MiB/s wr, 408 op/s 2024-09-20T22:21:37.825 INFO:teuthology.orchestra.run.smithi037.stdout: 5 106844 11.82 MB/sec execute 115 sec latency 26.927 ms 2024-09-20T22:21:37.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:37 smithi194 ceph-mon[25957]: pgmap v11: 97 pgs: 97 active+clean; 339 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 4.1 MiB/s wr, 408 op/s 2024-09-20T22:21:38.825 INFO:teuthology.orchestra.run.smithi037.stdout: 5 107243 11.77 MB/sec execute 116 sec latency 71.248 ms 2024-09-20T22:21:39.825 INFO:teuthology.orchestra.run.smithi037.stdout: 5 107586 11.74 MB/sec execute 117 sec latency 55.758 ms 2024-09-20T22:21:40.104 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:39 smithi037 ceph-mon[23718]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' 2024-09-20T22:21:40.105 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:39 smithi037 ceph-mon[23718]: pgmap v12: 97 pgs: 97 active+clean; 340 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 27 KiB/s rd, 7.7 MiB/s wr, 765 op/s 2024-09-20T22:21:40.105 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:39 smithi037 ceph-mon[23718]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' 2024-09-20T22:21:40.105 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:39 smithi037 ceph-mon[23718]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:21:40.105 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:39 smithi037 ceph-mon[23718]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:21:40.105 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:39 smithi037 ceph-mon[23718]: Updating smithi037:/etc/ceph/ceph.conf 2024-09-20T22:21:40.105 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:39 smithi037 ceph-mon[23718]: Updating smithi194:/etc/ceph/ceph.conf 2024-09-20T22:21:40.105 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:39 smithi037 ceph-mon[23718]: Updating smithi194:/var/lib/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/config/ceph.conf 2024-09-20T22:21:40.105 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:39 smithi037 ceph-mon[23718]: Updating smithi037:/var/lib/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/config/ceph.conf 2024-09-20T22:21:40.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:39 smithi194 ceph-mon[25957]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' 2024-09-20T22:21:40.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:39 smithi194 ceph-mon[25957]: pgmap v12: 97 pgs: 97 active+clean; 340 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 27 KiB/s rd, 7.7 MiB/s wr, 765 op/s 2024-09-20T22:21:40.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:39 smithi194 ceph-mon[25957]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' 2024-09-20T22:21:40.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:39 smithi194 ceph-mon[25957]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:21:40.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:39 smithi194 ceph-mon[25957]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:21:40.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:39 smithi194 ceph-mon[25957]: Updating smithi037:/etc/ceph/ceph.conf 2024-09-20T22:21:40.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:39 smithi194 ceph-mon[25957]: Updating smithi194:/etc/ceph/ceph.conf 2024-09-20T22:21:40.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:39 smithi194 ceph-mon[25957]: Updating smithi194:/var/lib/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/config/ceph.conf 2024-09-20T22:21:40.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:39 smithi194 ceph-mon[25957]: Updating smithi037:/var/lib/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/config/ceph.conf 2024-09-20T22:21:40.825 INFO:teuthology.orchestra.run.smithi037.stdout: 5 107937 11.67 MB/sec execute 118 sec latency 51.210 ms 2024-09-20T22:21:41.155 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:41 smithi037 ceph-mon[23718]: Updating smithi194:/etc/ceph/ceph.client.admin.keyring 2024-09-20T22:21:41.155 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:41 smithi037 ceph-mon[23718]: Updating smithi037:/etc/ceph/ceph.client.admin.keyring 2024-09-20T22:21:41.155 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:41 smithi037 ceph-mon[23718]: Updating smithi194:/var/lib/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/config/ceph.client.admin.keyring 2024-09-20T22:21:41.155 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:41 smithi037 ceph-mon[23718]: Updating smithi037:/var/lib/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/config/ceph.client.admin.keyring 2024-09-20T22:21:41.155 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:41 smithi037 ceph-mon[23718]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' 2024-09-20T22:21:41.155 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:41 smithi037 ceph-mon[23718]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' 2024-09-20T22:21:41.155 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:41 smithi037 ceph-mon[23718]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' 2024-09-20T22:21:41.155 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:41 smithi037 ceph-mon[23718]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' 2024-09-20T22:21:41.155 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:41 smithi037 ceph-mon[23718]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' 2024-09-20T22:21:41.155 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:41 smithi037 ceph-mon[23718]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' 2024-09-20T22:21:41.156 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:41 smithi037 ceph-mon[23718]: Reconfiguring prometheus.smithi037 (dependencies changed)... 2024-09-20T22:21:41.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:41 smithi194 ceph-mon[25957]: Updating smithi194:/etc/ceph/ceph.client.admin.keyring 2024-09-20T22:21:41.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:41 smithi194 ceph-mon[25957]: Updating smithi037:/etc/ceph/ceph.client.admin.keyring 2024-09-20T22:21:41.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:41 smithi194 ceph-mon[25957]: Updating smithi194:/var/lib/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/config/ceph.client.admin.keyring 2024-09-20T22:21:41.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:41 smithi194 ceph-mon[25957]: Updating smithi037:/var/lib/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/config/ceph.client.admin.keyring 2024-09-20T22:21:41.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:41 smithi194 ceph-mon[25957]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' 2024-09-20T22:21:41.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:41 smithi194 ceph-mon[25957]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' 2024-09-20T22:21:41.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:41 smithi194 ceph-mon[25957]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' 2024-09-20T22:21:41.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:41 smithi194 ceph-mon[25957]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' 2024-09-20T22:21:41.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:41 smithi194 ceph-mon[25957]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' 2024-09-20T22:21:41.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:41 smithi194 ceph-mon[25957]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' 2024-09-20T22:21:41.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:41 smithi194 ceph-mon[25957]: Reconfiguring prometheus.smithi037 (dependencies changed)... 2024-09-20T22:21:41.825 INFO:teuthology.orchestra.run.smithi037.stdout: 5 108314 11.66 MB/sec execute 119 sec latency 67.175 ms 2024-09-20T22:21:42.442 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:42 smithi037 ceph-mon[23718]: Reconfiguring daemon prometheus.smithi037 on smithi037 2024-09-20T22:21:42.443 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:42 smithi037 ceph-mon[23718]: pgmap v13: 97 pgs: 97 active+clean; 340 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 6.0 MiB/s wr, 542 op/s 2024-09-20T22:21:42.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:42 smithi194 ceph-mon[25957]: Reconfiguring daemon prometheus.smithi037 on smithi037 2024-09-20T22:21:42.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:42 smithi194 ceph-mon[25957]: pgmap v13: 97 pgs: 97 active+clean; 340 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 6.0 MiB/s wr, 542 op/s 2024-09-20T22:21:42.825 INFO:teuthology.orchestra.run.smithi037.stdout: 5 108846 11.70 MB/sec execute 120 sec latency 34.307 ms 2024-09-20T22:21:43.825 INFO:teuthology.orchestra.run.smithi037.stdout: 5 109342 11.70 MB/sec execute 121 sec latency 25.702 ms 2024-09-20T22:21:44.504 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:44 smithi037 ceph-mon[23718]: pgmap v14: 97 pgs: 97 active+clean; 325 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.2 MiB/s wr, 797 op/s 2024-09-20T22:21:44.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:44 smithi194 ceph-mon[25957]: pgmap v14: 97 pgs: 97 active+clean; 325 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.2 MiB/s wr, 797 op/s 2024-09-20T22:21:44.825 INFO:teuthology.orchestra.run.smithi037.stdout: 5 109838 11.78 MB/sec execute 122 sec latency 44.635 ms 2024-09-20T22:21:45.761 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:45 smithi037 ceph-mon[23718]: pgmap v15: 97 pgs: 97 active+clean; 325 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 5.8 MiB/s wr, 612 op/s 2024-09-20T22:21:45.825 INFO:teuthology.orchestra.run.smithi037.stdout: 5 110231 11.79 MB/sec execute 123 sec latency 29.987 ms 2024-09-20T22:21:45.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:45 smithi194 ceph-mon[25957]: pgmap v15: 97 pgs: 97 active+clean; 325 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 5.8 MiB/s wr, 612 op/s 2024-09-20T22:21:46.826 INFO:teuthology.orchestra.run.smithi037.stdout: 5 110549 11.76 MB/sec execute 124 sec latency 33.917 ms 2024-09-20T22:21:47.825 INFO:teuthology.orchestra.run.smithi037.stdout: 5 110880 11.69 MB/sec execute 125 sec latency 36.288 ms 2024-09-20T22:21:48.032 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:48 smithi037 ceph-mon[23718]: pgmap v16: 97 pgs: 97 active+clean; 325 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 5.8 MiB/s wr, 612 op/s 2024-09-20T22:21:48.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:48 smithi194 ceph-mon[25957]: pgmap v16: 97 pgs: 97 active+clean; 325 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 5.8 MiB/s wr, 612 op/s 2024-09-20T22:21:48.826 INFO:teuthology.orchestra.run.smithi037.stdout: 5 111256 11.67 MB/sec execute 126 sec latency 43.598 ms 2024-09-20T22:21:49.826 INFO:teuthology.orchestra.run.smithi037.stdout: 5 111683 11.67 MB/sec execute 127 sec latency 53.771 ms 2024-09-20T22:21:50.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:50 smithi194 ceph-mon[25957]: pgmap v17: 97 pgs: 97 active+clean; 337 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.8 MiB/s wr, 865 op/s 2024-09-20T22:21:50.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:50 smithi194 ceph-mon[25957]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' 2024-09-20T22:21:50.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:50 smithi194 ceph-mon[25957]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' 2024-09-20T22:21:50.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:50 smithi194 ceph-mon[25957]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-20T22:21:50.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:50 smithi194 ceph-mon[25957]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-20T22:21:50.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:50 smithi194 ceph-mon[25957]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:21:50.630 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:50 smithi037 ceph-mon[23718]: pgmap v17: 97 pgs: 97 active+clean; 337 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.8 MiB/s wr, 865 op/s 2024-09-20T22:21:50.630 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:50 smithi037 ceph-mon[23718]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' 2024-09-20T22:21:50.630 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:50 smithi037 ceph-mon[23718]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' 2024-09-20T22:21:50.631 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:50 smithi037 ceph-mon[23718]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-20T22:21:50.631 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:50 smithi037 ceph-mon[23718]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-20T22:21:50.631 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:50 smithi037 ceph-mon[23718]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:21:50.826 INFO:teuthology.orchestra.run.smithi037.stdout: 5 112247 11.66 MB/sec execute 128 sec latency 58.101 ms 2024-09-20T22:21:51.643 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:51 smithi037 ceph-mon[23718]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:21:51.643 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:51 smithi037 ceph-mon[23718]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi194.bkcglt", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-20T22:21:51.825 INFO:teuthology.orchestra.run.smithi037.stdout: 5 112873 11.72 MB/sec execute 129 sec latency 22.131 ms 2024-09-20T22:21:51.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:51 smithi194 ceph-mon[25957]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:21:51.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:51 smithi194 ceph-mon[25957]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi194.bkcglt", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-20T22:21:52.655 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:52 smithi037 ceph-mon[23718]: pgmap v18: 97 pgs: 97 active+clean; 337 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.2 MiB/s wr, 507 op/s 2024-09-20T22:21:52.655 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:52 smithi037 ceph-mon[23718]: Upgrade: Updating mgr.smithi194.bkcglt 2024-09-20T22:21:52.655 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:52 smithi037 ceph-mon[23718]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-20T22:21:52.656 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:52 smithi037 ceph-mon[23718]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:21:52.656 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:52 smithi037 ceph-mon[23718]: Deploying daemon mgr.smithi194.bkcglt on smithi194 2024-09-20T22:21:52.814 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:52 smithi194 ceph-mon[25957]: pgmap v18: 97 pgs: 97 active+clean; 337 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.2 MiB/s wr, 507 op/s 2024-09-20T22:21:52.814 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:52 smithi194 ceph-mon[25957]: Upgrade: Updating mgr.smithi194.bkcglt 2024-09-20T22:21:52.814 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:52 smithi194 ceph-mon[25957]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-20T22:21:52.814 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:52 smithi194 ceph-mon[25957]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:21:52.814 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:52 smithi194 ceph-mon[25957]: Deploying daemon mgr.smithi194.bkcglt on smithi194 2024-09-20T22:21:52.825 INFO:teuthology.orchestra.run.smithi037.stdout: 5 113569 11.82 MB/sec execute 130 sec latency 32.151 ms 2024-09-20T22:21:53.826 INFO:teuthology.orchestra.run.smithi037.stdout: 5 114211 11.88 MB/sec execute 131 sec latency 18.970 ms 2024-09-20T22:21:54.675 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:54 smithi037 ceph-mon[23718]: pgmap v19: 97 pgs: 97 active+clean; 330 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 7.8 MiB/s wr, 799 op/s 2024-09-20T22:21:54.826 INFO:teuthology.orchestra.run.smithi037.stdout: 5 114771 11.86 MB/sec execute 132 sec latency 32.014 ms 2024-09-20T22:21:54.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:54 smithi194 ceph-mon[25957]: pgmap v19: 97 pgs: 97 active+clean; 330 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 7.8 MiB/s wr, 799 op/s 2024-09-20T22:21:55.688 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:55 smithi037 ceph-mon[23718]: pgmap v20: 97 pgs: 97 active+clean; 330 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.6 MiB/s wr, 544 op/s 2024-09-20T22:21:55.695 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:55 smithi194 ceph-mon[25957]: pgmap v20: 97 pgs: 97 active+clean; 330 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.6 MiB/s wr, 544 op/s 2024-09-20T22:21:55.826 INFO:teuthology.orchestra.run.smithi037.stdout: 5 115316 11.87 MB/sec execute 133 sec latency 30.415 ms 2024-09-20T22:21:56.826 INFO:teuthology.orchestra.run.smithi037.stdout: 5 115769 11.85 MB/sec execute 134 sec latency 49.359 ms 2024-09-20T22:21:56.958 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:56 smithi037 ceph-mon[23718]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' 2024-09-20T22:21:56.958 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:56 smithi037 ceph-mon[23718]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' 2024-09-20T22:21:56.958 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:56 smithi037 ceph-mon[23718]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:21:56.958 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:56 smithi037 ceph-mon[23718]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' 2024-09-20T22:21:56.958 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:56 smithi037 ceph-mon[23718]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' 2024-09-20T22:21:57.034 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:56 smithi194 ceph-mon[25957]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' 2024-09-20T22:21:57.034 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:56 smithi194 ceph-mon[25957]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' 2024-09-20T22:21:57.034 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:56 smithi194 ceph-mon[25957]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:21:57.034 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:56 smithi194 ceph-mon[25957]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' 2024-09-20T22:21:57.034 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:56 smithi194 ceph-mon[25957]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' 2024-09-20T22:21:57.826 INFO:teuthology.orchestra.run.smithi037.stdout: 5 116230 11.85 MB/sec execute 135 sec latency 39.882 ms 2024-09-20T22:21:57.974 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:21:57 smithi037 ceph-mon[23718]: pgmap v21: 97 pgs: 97 active+clean; 330 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.6 MiB/s wr, 544 op/s 2024-09-20T22:21:58.004 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:21:57 smithi194 ceph-mon[25957]: pgmap v21: 97 pgs: 97 active+clean; 330 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.6 MiB/s wr, 544 op/s 2024-09-20T22:21:58.826 INFO:teuthology.orchestra.run.smithi037.stdout: 5 116624 11.89 MB/sec execute 136 sec latency 36.314 ms 2024-09-20T22:21:59.826 INFO:teuthology.orchestra.run.smithi037.stdout: 5 116960 11.89 MB/sec execute 137 sec latency 54.740 ms 2024-09-20T22:22:00.029 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:00 smithi037 ceph-mon[23718]: pgmap v22: 97 pgs: 97 active+clean; 328 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.9 MiB/s wr, 882 op/s 2024-09-20T22:22:00.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:00 smithi194 ceph-mon[25957]: pgmap v22: 97 pgs: 97 active+clean; 328 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.9 MiB/s wr, 882 op/s 2024-09-20T22:22:00.740 INFO:teuthology.orchestra.run.smithi037.stdout:true 2024-09-20T22:22:00.826 INFO:teuthology.orchestra.run.smithi037.stdout: 5 117272 11.89 MB/sec execute 138 sec latency 107.807 ms 2024-09-20T22:22:01.296 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:01 smithi037 ceph-mon[23718]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:01.296 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:01 smithi037 ceph-mon[23718]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:01.296 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:01 smithi037 ceph-mon[23718]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:01.296 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:01 smithi037 ceph-mon[23718]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:01.357 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:01 smithi194 ceph-mon[25957]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:01.357 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:01 smithi194 ceph-mon[25957]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:01.357 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:01 smithi194 ceph-mon[25957]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:01.357 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:01 smithi194 ceph-mon[25957]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:01.793 INFO:teuthology.orchestra.run.smithi037.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T22:22:01.793 INFO:teuthology.orchestra.run.smithi037.stdout:alertmanager.smithi037 smithi037 *:9093,9094 running (119s) 31s ago 10m 27.9M - 0.25.0 c8568f914cd2 d1f99b3cd707 2024-09-20T22:22:01.793 INFO:teuthology.orchestra.run.smithi037.stdout:crash.smithi037 smithi037 running (10m) 31s ago 10m 6400k - 17.2.0 e1d6a67b021e 88301022bdc8 2024-09-20T22:22:01.793 INFO:teuthology.orchestra.run.smithi037.stdout:crash.smithi194 smithi194 running (10m) 1s ago 10m 6405k - 17.2.0 e1d6a67b021e b553601d7ca5 2024-09-20T22:22:01.793 INFO:teuthology.orchestra.run.smithi037.stdout:grafana.smithi037 smithi037 *:3000 running (111s) 31s ago 10m 45.9M - dad864ee21e9 2306f3c2da19 2024-09-20T22:22:01.793 INFO:teuthology.orchestra.run.smithi037.stdout:mds.foofs.smithi037.sdtedi smithi037 running (4m) 31s ago 4m 36.9M - 17.2.0 e1d6a67b021e 0a7898d25b8d 2024-09-20T22:22:01.793 INFO:teuthology.orchestra.run.smithi037.stdout:mds.foofs.smithi194.lkroso smithi194 running (4m) 1s ago 4m 15.2M - 17.2.0 e1d6a67b021e 17736de30008 2024-09-20T22:22:01.793 INFO:teuthology.orchestra.run.smithi037.stdout:mgr.smithi037.lykblh smithi037 *:8443,9283,8765 running (66s) 31s ago 12m 474M - 19.1.1-367-g9e2ac70b cc9251b744a3 181e46fd9ba6 2024-09-20T22:22:01.794 INFO:teuthology.orchestra.run.smithi037.stdout:mgr.smithi194.bkcglt smithi194 *:8443,9283,8765 running (6s) 1s ago 10m 93.3M - 19.1.1-367-g9e2ac70b cc9251b744a3 1fa610c97517 2024-09-20T22:22:01.794 INFO:teuthology.orchestra.run.smithi037.stdout:mon.smithi037 smithi037 running (12m) 31s ago 12m 56.2M 2048M 17.2.0 e1d6a67b021e 71f591122786 2024-09-20T22:22:01.794 INFO:teuthology.orchestra.run.smithi037.stdout:mon.smithi194 smithi194 running (10m) 1s ago 10m 44.6M 2048M 17.2.0 e1d6a67b021e 0b8814ae6c66 2024-09-20T22:22:01.794 INFO:teuthology.orchestra.run.smithi037.stdout:nfs.foo.0.0.smithi037.spabdc smithi037 *:2049 running (4m) 31s ago 4m 262M - 4.0 e1d6a67b021e 858ff9d558a2 2024-09-20T22:22:01.794 INFO:teuthology.orchestra.run.smithi037.stdout:nfs.foo.1.0.smithi194.qzjyej smithi194 *:2049 running (4m) 1s ago 4m 87.7M - 4.0 e1d6a67b021e c07fe0eb7c75 2024-09-20T22:22:01.794 INFO:teuthology.orchestra.run.smithi037.stdout:node-exporter.smithi037 smithi037 *:9100 running (103s) 31s ago 10m 8560k - 1.5.0 0da6a335fe13 433e6aaf6137 2024-09-20T22:22:01.794 INFO:teuthology.orchestra.run.smithi037.stdout:node-exporter.smithi194 smithi194 *:9100 running (74s) 1s ago 9m 7972k - 1.5.0 0da6a335fe13 1d30bb810ad1 2024-09-20T22:22:01.794 INFO:teuthology.orchestra.run.smithi037.stdout:osd.0 smithi037 running (9m) 31s ago 9m 407M 1937M 17.2.0 e1d6a67b021e e54e84fd5030 2024-09-20T22:22:01.794 INFO:teuthology.orchestra.run.smithi037.stdout:osd.1 smithi037 running (8m) 31s ago 8m 302M 1937M 17.2.0 e1d6a67b021e 09647c133032 2024-09-20T22:22:01.794 INFO:teuthology.orchestra.run.smithi037.stdout:osd.2 smithi037 running (8m) 31s ago 8m 250M 1937M 17.2.0 e1d6a67b021e 6c15b1175256 2024-09-20T22:22:01.794 INFO:teuthology.orchestra.run.smithi037.stdout:osd.3 smithi037 running (7m) 31s ago 7m 371M 1937M 17.2.0 e1d6a67b021e 3a22952c6475 2024-09-20T22:22:01.794 INFO:teuthology.orchestra.run.smithi037.stdout:osd.4 smithi194 running (7m) 1s ago 7m 396M 2704M 17.2.0 e1d6a67b021e fa107be8915c 2024-09-20T22:22:01.794 INFO:teuthology.orchestra.run.smithi037.stdout:osd.5 smithi194 running (7m) 1s ago 7m 441M 2704M 17.2.0 e1d6a67b021e 290b21cbd131 2024-09-20T22:22:01.794 INFO:teuthology.orchestra.run.smithi037.stdout:osd.6 smithi194 running (6m) 1s ago 6m 382M 2704M 17.2.0 e1d6a67b021e 27f0720639ed 2024-09-20T22:22:01.795 INFO:teuthology.orchestra.run.smithi037.stdout:osd.7 smithi194 running (6m) 1s ago 6m 525M 2704M 17.2.0 e1d6a67b021e 62d163f4fd92 2024-09-20T22:22:01.795 INFO:teuthology.orchestra.run.smithi037.stdout:prometheus.smithi037 smithi037 *:9095 starting - - - - 2024-09-20T22:22:01.826 INFO:teuthology.orchestra.run.smithi037.stdout: 5 117659 11.86 MB/sec execute 139 sec latency 76.169 ms 2024-09-20T22:22:02.309 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:02 smithi037 ceph-mon[23718]: from='client.14836 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:22:02.310 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:02 smithi037 ceph-mon[23718]: pgmap v23: 97 pgs: 97 active+clean; 328 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.0 MiB/s wr, 630 op/s 2024-09-20T22:22:02.310 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:02 smithi037 ceph-mon[23718]: from='client.24615 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:22:02.310 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:02 smithi037 ceph-mon[23718]: Standby manager daemon smithi194.bkcglt restarted 2024-09-20T22:22:02.310 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:02 smithi037 ceph-mon[23718]: Standby manager daemon smithi194.bkcglt started 2024-09-20T22:22:02.310 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:02 smithi037 ceph-mon[23718]: from='mgr.? 172.21.15.194:0/3921164895' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi194.bkcglt/crt"}]: dispatch 2024-09-20T22:22:02.310 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:02 smithi037 ceph-mon[23718]: from='mgr.? 172.21.15.194:0/3921164895' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-20T22:22:02.310 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:02 smithi037 ceph-mon[23718]: from='mgr.? 172.21.15.194:0/3921164895' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi194.bkcglt/key"}]: dispatch 2024-09-20T22:22:02.310 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:02 smithi037 ceph-mon[23718]: from='mgr.? 172.21.15.194:0/3921164895' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-20T22:22:02.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:02 smithi194 ceph-mon[25957]: from='client.14836 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:22:02.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:02 smithi194 ceph-mon[25957]: pgmap v23: 97 pgs: 97 active+clean; 328 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.0 MiB/s wr, 630 op/s 2024-09-20T22:22:02.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:02 smithi194 ceph-mon[25957]: from='client.24615 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:22:02.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:02 smithi194 ceph-mon[25957]: Standby manager daemon smithi194.bkcglt restarted 2024-09-20T22:22:02.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:02 smithi194 ceph-mon[25957]: Standby manager daemon smithi194.bkcglt started 2024-09-20T22:22:02.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:02 smithi194 ceph-mon[25957]: from='mgr.? 172.21.15.194:0/3921164895' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi194.bkcglt/crt"}]: dispatch 2024-09-20T22:22:02.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:02 smithi194 ceph-mon[25957]: from='mgr.? 172.21.15.194:0/3921164895' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-20T22:22:02.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:02 smithi194 ceph-mon[25957]: from='mgr.? 172.21.15.194:0/3921164895' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi194.bkcglt/key"}]: dispatch 2024-09-20T22:22:02.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:02 smithi194 ceph-mon[25957]: from='mgr.? 172.21.15.194:0/3921164895' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-20T22:22:02.357 INFO:teuthology.orchestra.run.smithi037.stdout:{ 2024-09-20T22:22:02.357 INFO:teuthology.orchestra.run.smithi037.stdout: "mon": { 2024-09-20T22:22:02.357 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T22:22:02.357 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:22:02.358 INFO:teuthology.orchestra.run.smithi037.stdout: "mgr": { 2024-09-20T22:22:02.358 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 2 2024-09-20T22:22:02.358 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:22:02.358 INFO:teuthology.orchestra.run.smithi037.stdout: "osd": { 2024-09-20T22:22:02.358 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-20T22:22:02.358 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:22:02.358 INFO:teuthology.orchestra.run.smithi037.stdout: "mds": { 2024-09-20T22:22:02.358 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T22:22:02.358 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:22:02.358 INFO:teuthology.orchestra.run.smithi037.stdout: "overall": { 2024-09-20T22:22:02.358 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-09-20T22:22:02.358 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 2 2024-09-20T22:22:02.358 INFO:teuthology.orchestra.run.smithi037.stdout: } 2024-09-20T22:22:02.359 INFO:teuthology.orchestra.run.smithi037.stdout:} 2024-09-20T22:22:02.826 INFO:teuthology.orchestra.run.smithi037.stdout: 5 117902 11.80 MB/sec execute 140 sec latency 46.883 ms 2024-09-20T22:22:02.962 INFO:teuthology.orchestra.run.smithi037.stdout:{ 2024-09-20T22:22:02.962 INFO:teuthology.orchestra.run.smithi037.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5", 2024-09-20T22:22:02.962 INFO:teuthology.orchestra.run.smithi037.stdout: "in_progress": true, 2024-09-20T22:22:02.962 INFO:teuthology.orchestra.run.smithi037.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-20T22:22:02.962 INFO:teuthology.orchestra.run.smithi037.stdout: "services_complete": [ 2024-09-20T22:22:02.962 INFO:teuthology.orchestra.run.smithi037.stdout: "mgr" 2024-09-20T22:22:02.962 INFO:teuthology.orchestra.run.smithi037.stdout: ], 2024-09-20T22:22:02.962 INFO:teuthology.orchestra.run.smithi037.stdout: "progress": "2/23 daemons upgraded", 2024-09-20T22:22:02.962 INFO:teuthology.orchestra.run.smithi037.stdout: "message": "Currently upgrading mgr daemons", 2024-09-20T22:22:02.962 INFO:teuthology.orchestra.run.smithi037.stdout: "is_paused": false 2024-09-20T22:22:02.962 INFO:teuthology.orchestra.run.smithi037.stdout:} 2024-09-20T22:22:03.320 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:03 smithi037 ceph-mon[23718]: from='client.14842 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:22:03.321 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:03 smithi037 ceph-mon[23718]: from='client.? 172.21.15.37:0/3503866108' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:22:03.321 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:03 smithi037 ceph-mon[23718]: mgrmap e36: smithi037.lykblh(active, since 42s), standbys: smithi194.bkcglt 2024-09-20T22:22:03.527 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:03 smithi194 ceph-mon[25957]: from='client.14842 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:22:03.527 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:03 smithi194 ceph-mon[25957]: from='client.? 172.21.15.37:0/3503866108' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:22:03.527 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:03 smithi194 ceph-mon[25957]: mgrmap e36: smithi037.lykblh(active, since 42s), standbys: smithi194.bkcglt 2024-09-20T22:22:03.647 INFO:teuthology.orchestra.run.smithi037.stdout:HEALTH_OK 2024-09-20T22:22:03.826 INFO:teuthology.orchestra.run.smithi037.stdout: 5 118243 11.77 MB/sec execute 141 sec latency 53.458 ms 2024-09-20T22:22:04.236 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:04 smithi194 ceph-mon[25957]: from='client.14850 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:22:04.236 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:04 smithi194 ceph-mon[25957]: pgmap v24: 97 pgs: 97 active+clean; 338 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.7 MiB/s wr, 847 op/s 2024-09-20T22:22:04.237 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:04 smithi194 ceph-mon[25957]: from='client.? 172.21.15.37:0/416065951' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T22:22:04.335 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:04 smithi037 ceph-mon[23718]: from='client.14850 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:22:04.335 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:04 smithi037 ceph-mon[23718]: pgmap v24: 97 pgs: 97 active+clean; 338 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.7 MiB/s wr, 847 op/s 2024-09-20T22:22:04.335 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:04 smithi037 ceph-mon[23718]: from='client.? 172.21.15.37:0/416065951' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T22:22:04.826 INFO:teuthology.orchestra.run.smithi037.stdout: 5 118691 11.74 MB/sec execute 142 sec latency 39.699 ms 2024-09-20T22:22:05.344 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:05 smithi037 ceph-mon[23718]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:05.345 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:05 smithi037 ceph-mon[23718]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:05.345 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:05 smithi037 ceph-mon[23718]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:05.345 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:05 smithi037 ceph-mon[23718]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:05.345 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:05 smithi037 ceph-mon[23718]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:22:05.345 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:05 smithi037 ceph-mon[23718]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:22:05.345 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:05 smithi037 ceph-mon[23718]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:05.345 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:05 smithi037 ceph-mon[23718]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:05.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:05 smithi194 ceph-mon[25957]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:05.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:05 smithi194 ceph-mon[25957]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:05.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:05 smithi194 ceph-mon[25957]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:05.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:05 smithi194 ceph-mon[25957]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:05.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:05 smithi194 ceph-mon[25957]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:22:05.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:05 smithi194 ceph-mon[25957]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:22:05.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:05 smithi194 ceph-mon[25957]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:05.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:05 smithi194 ceph-mon[25957]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:05.826 INFO:teuthology.orchestra.run.smithi037.stdout: 5 119100 11.74 MB/sec execute 143 sec latency 42.879 ms 2024-09-20T22:22:06.350 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:06 smithi037 ceph-mon[23718]: pgmap v25: 97 pgs: 97 active+clean; 338 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.9 KiB/s rd, 6.1 MiB/s wr, 555 op/s 2024-09-20T22:22:06.351 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:06 smithi037 ceph-mon[23718]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:22:06.351 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:06 smithi037 ceph-mon[23718]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:22:06.351 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:06 smithi037 ceph-mon[23718]: Upgrade: Setting container_image for all mgr 2024-09-20T22:22:06.351 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:06 smithi037 ceph-mon[23718]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:06.351 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:06 smithi037 ceph-mon[23718]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi037.lykblh"}]: dispatch 2024-09-20T22:22:06.351 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:06 smithi037 ceph-mon[23718]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi037.lykblh"}]': finished 2024-09-20T22:22:06.351 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:06 smithi037 ceph-mon[23718]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi194.bkcglt"}]: dispatch 2024-09-20T22:22:06.351 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:06 smithi037 ceph-mon[23718]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi194.bkcglt"}]': finished 2024-09-20T22:22:06.351 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:06 smithi037 ceph-mon[23718]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-20T22:22:06.351 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:06 smithi037 ceph-mon[23718]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:22:06.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:06 smithi194 ceph-mon[25957]: pgmap v25: 97 pgs: 97 active+clean; 338 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.9 KiB/s rd, 6.1 MiB/s wr, 555 op/s 2024-09-20T22:22:06.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:06 smithi194 ceph-mon[25957]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:22:06.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:06 smithi194 ceph-mon[25957]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:22:06.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:06 smithi194 ceph-mon[25957]: Upgrade: Setting container_image for all mgr 2024-09-20T22:22:06.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:06 smithi194 ceph-mon[25957]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:06.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:06 smithi194 ceph-mon[25957]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi037.lykblh"}]: dispatch 2024-09-20T22:22:06.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:06 smithi194 ceph-mon[25957]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi037.lykblh"}]': finished 2024-09-20T22:22:06.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:06 smithi194 ceph-mon[25957]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi194.bkcglt"}]: dispatch 2024-09-20T22:22:06.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:06 smithi194 ceph-mon[25957]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi194.bkcglt"}]': finished 2024-09-20T22:22:06.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:06 smithi194 ceph-mon[25957]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-20T22:22:06.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:06 smithi194 ceph-mon[25957]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:22:06.826 INFO:teuthology.orchestra.run.smithi037.stdout: 5 119759 11.77 MB/sec execute 144 sec latency 36.144 ms 2024-09-20T22:22:07.826 INFO:teuthology.orchestra.run.smithi037.stdout: 5 120204 11.82 MB/sec execute 145 sec latency 34.941 ms 2024-09-20T22:22:08.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:08 smithi194 ceph-mon[25957]: pgmap v26: 97 pgs: 97 active+clean; 337 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.9 KiB/s rd, 6.6 MiB/s wr, 639 op/s 2024-09-20T22:22:08.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:08 smithi194 ceph-mon[25957]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:08.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:08 smithi194 ceph-mon[25957]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-20T22:22:08.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:08 smithi194 ceph-mon[25957]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-20T22:22:08.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:08 smithi194 ceph-mon[25957]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:22:08.388 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:08 smithi037 ceph-mon[23718]: pgmap v26: 97 pgs: 97 active+clean; 337 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.9 KiB/s rd, 6.6 MiB/s wr, 639 op/s 2024-09-20T22:22:08.388 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:08 smithi037 ceph-mon[23718]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:08.388 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:08 smithi037 ceph-mon[23718]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-20T22:22:08.388 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:08 smithi037 ceph-mon[23718]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-20T22:22:08.388 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:08 smithi037 ceph-mon[23718]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:22:08.826 INFO:teuthology.orchestra.run.smithi037.stdout: 5 120903 11.91 MB/sec execute 146 sec latency 24.470 ms 2024-09-20T22:22:09.392 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:09 smithi037 ceph-mon[23718]: Upgrade: Updating mon.smithi037 2024-09-20T22:22:09.393 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:09 smithi037 ceph-mon[23718]: Deploying daemon mon.smithi037 on smithi037 2024-09-20T22:22:09.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:09 smithi194 ceph-mon[25957]: Upgrade: Updating mon.smithi037 2024-09-20T22:22:09.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:09 smithi194 ceph-mon[25957]: Deploying daemon mon.smithi037 on smithi037 2024-09-20T22:22:09.826 INFO:teuthology.orchestra.run.smithi037.stdout: 5 121279 11.88 MB/sec execute 147 sec latency 33.207 ms 2024-09-20T22:22:10.405 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:10 smithi037 ceph-mon[23718]: pgmap v27: 97 pgs: 97 active+clean; 328 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.3 MiB/s wr, 812 op/s 2024-09-20T22:22:10.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:10 smithi194 ceph-mon[25957]: pgmap v27: 97 pgs: 97 active+clean; 328 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.3 MiB/s wr, 812 op/s 2024-09-20T22:22:10.826 INFO:teuthology.orchestra.run.smithi037.stdout: 5 121669 11.86 MB/sec execute 148 sec latency 47.615 ms 2024-09-20T22:22:11.439 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:11 smithi037 systemd[1]: Stopping Ceph mon.smithi037 for f9a6357a-779c-11ef-baf5-efdc52797490... 2024-09-20T22:22:11.700 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:11 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-mon-smithi037[23714]: 2024-09-20T22:22:11.671+0000 7f46fe71e700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi037 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-20T22:22:11.700 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:11 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-mon-smithi037[23714]: 2024-09-20T22:22:11.671+0000 7f46fe71e700 -1 mon.smithi037@0(leader) e2 *** Got Signal Terminated *** 2024-09-20T22:22:11.826 INFO:teuthology.orchestra.run.smithi037.stdout: 5 122076 11.80 MB/sec execute 149 sec latency 47.942 ms 2024-09-20T22:22:11.951 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:11 smithi037 podman[65469]: 2024-09-20 22:22:11.836776258 +0000 UTC m=+0.387753045 container died 71f591122786ae60b80a25133890317cf9cad97c3c7ba98976cff6c0bb977f18 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-mon-smithi037, version=8, io.openshift.tags=base centos centos-stream, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, vcs-type=git, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, GIT_CLEAN=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD) 2024-09-20T22:22:12.619 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:12 smithi037 podman[65469]: 2024-09-20 22:22:12.611333017 +0000 UTC m=+1.162309796 container cleanup 71f591122786ae60b80a25133890317cf9cad97c3c7ba98976cff6c0bb977f18 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-mon-smithi037, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , GIT_CLEAN=True, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, io.openshift.expose-services=, version=8, io.buildah.version=1.19.8, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, release=754, build-date=2022-05-03T08:36:31.336870, ceph=True, name=centos-stream, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-09-20T22:22:12.619 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:12 smithi037 bash[65469]: ceph-f9a6357a-779c-11ef-baf5-efdc52797490-mon-smithi037 2024-09-20T22:22:12.826 INFO:teuthology.orchestra.run.smithi037.stdout: 5 122871 11.89 MB/sec execute 150 sec latency 28.953 ms 2024-09-20T22:22:13.007 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:12 smithi037 podman[65482]: 2024-09-20 22:22:12.95632247 +0000 UTC m=+1.115566730 container remove 71f591122786ae60b80a25133890317cf9cad97c3c7ba98976cff6c0bb977f18 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-mon-smithi037, GIT_CLEAN=True, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, architecture=x86_64, com.redhat.component=centos-stream-container, RELEASE=HEAD, distribution-scope=public, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, version=8, release=754, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8) 2024-09-20T22:22:13.510 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:13 smithi037 systemd[1]: ceph-f9a6357a-779c-11ef-baf5-efdc52797490@mon.smithi037.service: Deactivated successfully. 2024-09-20T22:22:13.511 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:13 smithi037 systemd[1]: Stopped Ceph mon.smithi037 for f9a6357a-779c-11ef-baf5-efdc52797490. 2024-09-20T22:22:13.511 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:13 smithi037 systemd[1]: ceph-f9a6357a-779c-11ef-baf5-efdc52797490@mon.smithi037.service: Consumed 17.399s CPU time. 2024-09-20T22:22:13.761 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:13 smithi037 systemd[1]: Starting Ceph mon.smithi037 for f9a6357a-779c-11ef-baf5-efdc52797490... 2024-09-20T22:22:13.826 INFO:teuthology.orchestra.run.smithi037.stdout: 5 123548 11.92 MB/sec execute 151 sec latency 43.954 ms 2024-09-20T22:22:14.013 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 podman[65621]: 2024-09-20 22:22:13.903324823 +0000 UTC m=+0.022801163 image pull cc9251b744a3faafca7c3095f79bfff81873afe823f31a03a11e015a34abcdbd quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5 2024-09-20T22:22:14.268 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 podman[65621]: 2024-09-20 22:22:14.054184078 +0000 UTC m=+0.173660406 container create c0229eceb527050ba8644ebd6d5faff53bf9e2c05aec6ccdd04a9821ec899cf7 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-mon-smithi037, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=squid-9e2ac70, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-20T22:22:14.527 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 podman[65621]: 2024-09-20 22:22:14.387293478 +0000 UTC m=+0.506769806 container init c0229eceb527050ba8644ebd6d5faff53bf9e2c05aec6ccdd04a9821ec899cf7 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-mon-smithi037, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, RELEASE=squid-9e2ac70, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-09-20T22:22:14.528 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 podman[65621]: 2024-09-20 22:22:14.392938152 +0000 UTC m=+0.512414488 container start c0229eceb527050ba8644ebd6d5faff53bf9e2c05aec6ccdd04a9821ec899cf7 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-mon-smithi037, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, RELEASE=squid-9e2ac70, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-09-20T22:22:14.528 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: set uid:gid to 167:167 (ceph:ceph) 2024-09-20T22:22:14.528 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable), process ceph-mon, pid 2 2024-09-20T22:22:14.528 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: pidfile_write: ignore empty --pid-file 2024-09-20T22:22:14.528 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: load: jerasure load: lrc 2024-09-20T22:22:14.528 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: RocksDB version: 7.9.2 2024-09-20T22:22:14.528 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Git sha 0 2024-09-20T22:22:14.528 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Compile date 2024-09-19 18:20:48 2024-09-20T22:22:14.528 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: DB SUMMARY 2024-09-20T22:22:14.528 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: DB Session ID: YLCW2Q8J29TXPBB225IJ 2024-09-20T22:22:14.528 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: CURRENT file: CURRENT 2024-09-20T22:22:14.528 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: IDENTITY file: IDENTITY 2024-09-20T22:22:14.528 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: MANIFEST file: MANIFEST-000015 size: 980 Bytes 2024-09-20T22:22:14.529 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi037/store.db dir, Total Num: 1, files: 000030.sst 2024-09-20T22:22:14.529 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi037/store.db: 000028.log size: 1347189 ; 2024-09-20T22:22:14.529 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.error_if_exists: 0 2024-09-20T22:22:14.529 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.create_if_missing: 0 2024-09-20T22:22:14.529 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.paranoid_checks: 1 2024-09-20T22:22:14.529 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-20T22:22:14.529 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-20T22:22:14.529 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-20T22:22:14.529 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.env: 0x560e57a5e940 2024-09-20T22:22:14.529 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.fs: PosixFileSystem 2024-09-20T22:22:14.529 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.info_log: 0x560e597d38c0 2024-09-20T22:22:14.529 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.max_file_opening_threads: 16 2024-09-20T22:22:14.530 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.statistics: (nil) 2024-09-20T22:22:14.530 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.use_fsync: 0 2024-09-20T22:22:14.530 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.max_log_file_size: 0 2024-09-20T22:22:14.530 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-20T22:22:14.530 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-20T22:22:14.530 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.keep_log_file_num: 1000 2024-09-20T22:22:14.530 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.recycle_log_file_num: 0 2024-09-20T22:22:14.530 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.allow_fallocate: 1 2024-09-20T22:22:14.530 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.allow_mmap_reads: 0 2024-09-20T22:22:14.530 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.allow_mmap_writes: 0 2024-09-20T22:22:14.531 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.use_direct_reads: 0 2024-09-20T22:22:14.531 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-20T22:22:14.531 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.create_missing_column_families: 0 2024-09-20T22:22:14.531 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.db_log_dir: 2024-09-20T22:22:14.531 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.wal_dir: 2024-09-20T22:22:14.531 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-20T22:22:14.531 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-20T22:22:14.531 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-20T22:22:14.531 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-20T22:22:14.531 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-20T22:22:14.531 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-20T22:22:14.531 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.advise_random_on_open: 1 2024-09-20T22:22:14.531 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.db_write_buffer_size: 0 2024-09-20T22:22:14.531 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.write_buffer_manager: 0x560e597e2dc0 2024-09-20T22:22:14.532 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-20T22:22:14.532 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-20T22:22:14.532 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-20T22:22:14.532 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.rate_limiter: (nil) 2024-09-20T22:22:14.532 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-20T22:22:14.532 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.wal_recovery_mode: 2 2024-09-20T22:22:14.532 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.enable_thread_tracking: 0 2024-09-20T22:22:14.532 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.enable_pipelined_write: 0 2024-09-20T22:22:14.532 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.unordered_write: 0 2024-09-20T22:22:14.532 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-20T22:22:14.532 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-20T22:22:14.532 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-20T22:22:14.532 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-20T22:22:14.532 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.row_cache: None 2024-09-20T22:22:14.532 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.wal_filter: None 2024-09-20T22:22:14.533 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-20T22:22:14.533 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.allow_ingest_behind: 0 2024-09-20T22:22:14.533 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.two_write_queues: 0 2024-09-20T22:22:14.533 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.manual_wal_flush: 0 2024-09-20T22:22:14.533 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.wal_compression: 0 2024-09-20T22:22:14.533 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.atomic_flush: 0 2024-09-20T22:22:14.533 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-20T22:22:14.533 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-20T22:22:14.533 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-20T22:22:14.533 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.log_readahead_size: 0 2024-09-20T22:22:14.533 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-20T22:22:14.533 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.best_efforts_recovery: 0 2024-09-20T22:22:14.533 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-20T22:22:14.533 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-20T22:22:14.533 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.allow_data_in_errors: 0 2024-09-20T22:22:14.534 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.db_host_id: __hostname__ 2024-09-20T22:22:14.534 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-20T22:22:14.534 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.max_background_jobs: 2 2024-09-20T22:22:14.534 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.max_background_compactions: -1 2024-09-20T22:22:14.534 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.max_subcompactions: 1 2024-09-20T22:22:14.534 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-20T22:22:14.534 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-20T22:22:14.534 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-20T22:22:14.534 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.max_total_wal_size: 0 2024-09-20T22:22:14.534 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-20T22:22:14.534 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-20T22:22:14.534 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-20T22:22:14.534 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-20T22:22:14.534 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.max_open_files: -1 2024-09-20T22:22:14.535 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.bytes_per_sync: 0 2024-09-20T22:22:14.535 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-20T22:22:14.535 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-20T22:22:14.535 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.compaction_readahead_size: 0 2024-09-20T22:22:14.535 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.max_background_flushes: -1 2024-09-20T22:22:14.535 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Compression algorithms supported: 2024-09-20T22:22:14.535 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: kZSTD supported: 0 2024-09-20T22:22:14.535 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: kXpressCompression supported: 0 2024-09-20T22:22:14.535 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: kBZip2Compression supported: 0 2024-09-20T22:22:14.535 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-20T22:22:14.535 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: kLZ4Compression supported: 1 2024-09-20T22:22:14.535 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: kZlibCompression supported: 1 2024-09-20T22:22:14.535 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: kLZ4HCCompression supported: 1 2024-09-20T22:22:14.536 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: kSnappyCompression supported: 1 2024-09-20T22:22:14.536 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-20T22:22:14.536 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-20T22:22:14.536 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi037/store.db/MANIFEST-000015 2024-09-20T22:22:14.537 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-20T22:22:14.537 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-20T22:22:14.537 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.merge_operator: 2024-09-20T22:22:14.537 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.compaction_filter: None 2024-09-20T22:22:14.537 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.compaction_filter_factory: None 2024-09-20T22:22:14.537 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.sst_partitioner_factory: None 2024-09-20T22:22:14.537 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-20T22:22:14.537 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-20T22:22:14.537 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x560e597d3520) 2024-09-20T22:22:14.537 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: cache_index_and_filter_blocks: 1 2024-09-20T22:22:14.537 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-20T22:22:14.537 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-20T22:22:14.537 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: pin_top_level_index_and_filter: 1 2024-09-20T22:22:14.537 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: index_type: 0 2024-09-20T22:22:14.537 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: data_block_index_type: 0 2024-09-20T22:22:14.538 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: index_shortening: 1 2024-09-20T22:22:14.538 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-20T22:22:14.538 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: checksum: 4 2024-09-20T22:22:14.538 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: no_block_cache: 0 2024-09-20T22:22:14.538 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: block_cache: 0x560e597c71f0 2024-09-20T22:22:14.538 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: block_cache_name: BinnedLRUCache 2024-09-20T22:22:14.538 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: block_cache_options: 2024-09-20T22:22:14.538 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: capacity : 536870912 2024-09-20T22:22:14.538 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: num_shard_bits : 4 2024-09-20T22:22:14.538 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: strict_capacity_limit : 0 2024-09-20T22:22:14.538 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: high_pri_pool_ratio: 0.000 2024-09-20T22:22:14.538 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: block_cache_compressed: (nil) 2024-09-20T22:22:14.538 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: persistent_cache: (nil) 2024-09-20T22:22:14.538 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: block_size: 4096 2024-09-20T22:22:14.539 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: block_size_deviation: 10 2024-09-20T22:22:14.539 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: block_restart_interval: 16 2024-09-20T22:22:14.539 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: index_block_restart_interval: 1 2024-09-20T22:22:14.539 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: metadata_block_size: 4096 2024-09-20T22:22:14.539 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: partition_filters: 0 2024-09-20T22:22:14.539 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: use_delta_encoding: 1 2024-09-20T22:22:14.539 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: filter_policy: bloomfilter 2024-09-20T22:22:14.539 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: whole_key_filtering: 1 2024-09-20T22:22:14.539 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: verify_compression: 0 2024-09-20T22:22:14.539 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: read_amp_bytes_per_bit: 0 2024-09-20T22:22:14.539 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: format_version: 5 2024-09-20T22:22:14.539 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: enable_index_compression: 1 2024-09-20T22:22:14.539 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: block_align: 0 2024-09-20T22:22:14.539 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: max_auto_readahead_size: 262144 2024-09-20T22:22:14.540 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: prepopulate_block_cache: 0 2024-09-20T22:22:14.540 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: initial_auto_readahead_size: 8192 2024-09-20T22:22:14.540 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: num_file_reads_for_auto_readahead: 2 2024-09-20T22:22:14.540 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.write_buffer_size: 33554432 2024-09-20T22:22:14.540 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.max_write_buffer_number: 2 2024-09-20T22:22:14.540 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.compression: NoCompression 2024-09-20T22:22:14.540 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.bottommost_compression: Disabled 2024-09-20T22:22:14.540 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.prefix_extractor: nullptr 2024-09-20T22:22:14.540 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-20T22:22:14.540 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.num_levels: 7 2024-09-20T22:22:14.540 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-20T22:22:14.540 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-20T22:22:14.540 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-20T22:22:14.540 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-20T22:22:14.540 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-20T22:22:14.541 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-20T22:22:14.541 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-20T22:22:14.541 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-20T22:22:14.541 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-20T22:22:14.541 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-20T22:22:14.541 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-20T22:22:14.541 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-20T22:22:14.541 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-20T22:22:14.541 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.compression_opts.level: 32767 2024-09-20T22:22:14.541 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.compression_opts.strategy: 0 2024-09-20T22:22:14.541 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-20T22:22:14.541 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-20T22:22:14.541 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-20T22:22:14.541 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-20T22:22:14.541 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.compression_opts.enabled: false 2024-09-20T22:22:14.542 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-20T22:22:14.542 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-20T22:22:14.542 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-20T22:22:14.542 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-20T22:22:14.542 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.target_file_size_base: 67108864 2024-09-20T22:22:14.542 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-20T22:22:14.542 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-20T22:22:14.542 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-20T22:22:14.542 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-20T22:22:14.542 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-20T22:22:14.542 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-20T22:22:14.542 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-20T22:22:14.542 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-20T22:22:14.542 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-20T22:22:14.542 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-20T22:22:14.543 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-20T22:22:14.543 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-20T22:22:14.543 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-20T22:22:14.543 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-20T22:22:14.543 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.arena_block_size: 1048576 2024-09-20T22:22:14.543 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-20T22:22:14.543 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-20T22:22:14.543 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.disable_auto_compactions: 0 2024-09-20T22:22:14.543 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-20T22:22:14.543 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-20T22:22:14.543 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-20T22:22:14.543 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-20T22:22:14.543 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-20T22:22:14.543 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-20T22:22:14.544 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-20T22:22:14.544 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-20T22:22:14.544 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-20T22:22:14.544 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-20T22:22:14.544 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-20T22:22:14.544 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.inplace_update_support: 0 2024-09-20T22:22:14.544 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-20T22:22:14.544 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-20T22:22:14.544 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-20T22:22:14.544 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-20T22:22:14.544 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.bloom_locality: 0 2024-09-20T22:22:14.544 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.max_successive_merges: 0 2024-09-20T22:22:14.544 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-20T22:22:14.544 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.paranoid_file_checks: 0 2024-09-20T22:22:14.544 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.force_consistency_checks: 1 2024-09-20T22:22:14.545 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.report_bg_io_stats: 0 2024-09-20T22:22:14.545 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.ttl: 2592000 2024-09-20T22:22:14.545 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-20T22:22:14.545 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-20T22:22:14.545 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-20T22:22:14.545 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.enable_blob_files: false 2024-09-20T22:22:14.545 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.min_blob_size: 0 2024-09-20T22:22:14.545 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.blob_file_size: 268435456 2024-09-20T22:22:14.545 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-20T22:22:14.545 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-20T22:22:14.545 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-20T22:22:14.545 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-20T22:22:14.545 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-20T22:22:14.545 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.blob_file_starting_level: 0 2024-09-20T22:22:14.545 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-20T22:22:14.546 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 30.sst 2024-09-20T22:22:14.546 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-20T22:22:14.546 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi037/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 32, last_sequence is 11890, log_number is 28,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-20T22:22:14.546 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 28 2024-09-20T22:22:14.546 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: c3fa949c-238a-4d0b-8729-046e6c8205d9 2024-09-20T22:22:14.546 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726870934423174, "job": 1, "event": "recovery_started", "wal_files": [28]} 2024-09-20T22:22:14.546 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #28 mode 2 2024-09-20T22:22:14.546 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726870934428586, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 33, "file_size": 1180510, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 11891, "largest_seqno": 12730, "table_properties": {"data_size": 1176129, "index_size": 2445, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 901, "raw_key_size": 8121, "raw_average_key_size": 23, "raw_value_size": 1168334, "raw_average_value_size": 3319, "num_data_blocks": 112, "num_entries": 352, "num_filter_entries": 352, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726870934, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "c3fa949c-238a-4d0b-8729-046e6c8205d9", "db_session_id": "YLCW2Q8J29TXPBB225IJ", "orig_file_number": 33, "seqno_to_time_mapping": "N/A"}} 2024-09-20T22:22:14.546 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726870934428753, "job": 1, "event": "recovery_finished"} 2024-09-20T22:22:14.546 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: [db/version_set.cc:5047] Creating manifest 35 2024-09-20T22:22:14.546 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-20T22:22:14.546 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-smithi037/store.db/000028.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-20T22:22:14.546 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x560e597f4e00 2024-09-20T22:22:14.546 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: DB pointer 0x560e598f0000 2024-09-20T22:22:14.547 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-20T22:22:14.547 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-20T22:22:14.547 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: ** DB Stats ** 2024-09-20T22:22:14.547 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-20T22:22:14.547 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-20T22:22:14.547 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-20T22:22:14.547 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-20T22:22:14.547 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-20T22:22:14.547 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-20T22:22:14.547 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-20T22:22:14.547 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: 2024-09-20T22:22:14.547 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: ** Compaction Stats [default] ** 2024-09-20T22:22:14.547 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-20T22:22:14.547 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-20T22:22:14.548 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: L0 1/0 1.13 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 354.8 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-09-20T22:22:14.548 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: L6 1/0 8.14 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-20T22:22:14.548 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: Sum 2/0 9.26 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 354.8 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-09-20T22:22:14.548 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 354.8 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-09-20T22:22:14.548 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: 2024-09-20T22:22:14.548 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: ** Compaction Stats [default] ** 2024-09-20T22:22:14.548 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-20T22:22:14.548 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-20T22:22:14.548 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 354.8 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-09-20T22:22:14.548 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: 2024-09-20T22:22:14.548 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-20T22:22:14.548 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: 2024-09-20T22:22:14.548 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-20T22:22:14.548 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: Flush(GB): cumulative 0.001, interval 0.001 2024-09-20T22:22:14.548 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-20T22:22:14.549 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-20T22:22:14.549 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-20T22:22:14.549 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-20T22:22:14.549 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: Cumulative compaction: 0.00 GB write, 110.95 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-20T22:22:14.549 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: Interval compaction: 0.00 GB write, 110.95 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-20T22:22:14.549 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-20T22:22:14.549 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: Block cache BinnedLRUCache@0x560e597c71f0#2 capacity: 512.00 MB usage: 3.61 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2024-09-20T22:22:14.549 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.03 KB,0.000196695%) IndexBlock(1,2.58 KB,0.000491738%) Misc(1,0.00 KB,0%) 2024-09-20T22:22:14.549 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: 2024-09-20T22:22:14.549 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-20T22:22:14.549 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: starting mon.smithi037 rank 0 at public addrs [v2:172.21.15.37:3300/0,v1:172.21.15.37:6789/0] at bind addrs [v2:172.21.15.37:3300/0,v1:172.21.15.37:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi037 fsid f9a6357a-779c-11ef-baf5-efdc52797490 2024-09-20T22:22:14.549 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: mon.smithi037@-1(???) e2 preinit fsid f9a6357a-779c-11ef-baf5-efdc52797490 2024-09-20T22:22:14.549 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: mon.smithi037@-1(???).mds e7 new map 2024-09-20T22:22:14.549 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: mon.smithi037@-1(???).mds e7 print_map 2024-09-20T22:22:14.549 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: e7 2024-09-20T22:22:14.550 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-20T22:22:14.550 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-20T22:22:14.550 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-20T22:22:14.550 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: legacy client fscid: 1 2024-09-20T22:22:14.550 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: 2024-09-20T22:22:14.550 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: Filesystem 'foofs' (1) 2024-09-20T22:22:14.550 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: fs_name foofs 2024-09-20T22:22:14.550 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: epoch 5 2024-09-20T22:22:14.550 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-20T22:22:14.550 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: created 2024-09-20T22:17:05.179585+0000 2024-09-20T22:22:14.550 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: modified 2024-09-20T22:17:11.201678+0000 2024-09-20T22:22:14.550 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: tableserver 0 2024-09-20T22:22:14.550 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: root 0 2024-09-20T22:22:14.550 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: session_timeout 60 2024-09-20T22:22:14.551 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: session_autoclose 300 2024-09-20T22:22:14.551 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: max_file_size 1099511627776 2024-09-20T22:22:14.551 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: max_xattr_size 65536 2024-09-20T22:22:14.551 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: required_client_features {} 2024-09-20T22:22:14.551 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: last_failure 0 2024-09-20T22:22:14.551 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: last_failure_osd_epoch 0 2024-09-20T22:22:14.551 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-20T22:22:14.551 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: max_mds 1 2024-09-20T22:22:14.551 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: in 0 2024-09-20T22:22:14.551 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: up {0=14488} 2024-09-20T22:22:14.551 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: failed 2024-09-20T22:22:14.551 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: damaged 2024-09-20T22:22:14.551 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: stopped 2024-09-20T22:22:14.551 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: data_pools [3] 2024-09-20T22:22:14.551 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: metadata_pool 2 2024-09-20T22:22:14.552 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: inline_data disabled 2024-09-20T22:22:14.552 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: balancer 2024-09-20T22:22:14.552 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: bal_rank_mask -1 2024-09-20T22:22:14.552 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: standby_count_wanted 1 2024-09-20T22:22:14.552 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: qdb_cluster leader: 0 members: 2024-09-20T22:22:14.552 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: [mds.foofs.smithi037.sdtedi{0:14488} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.37:6834/2142770746,v1:172.21.15.37:6835/2142770746] compat {c=[1],r=[1],i=[7ff]}] 2024-09-20T22:22:14.552 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: 2024-09-20T22:22:14.552 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: 2024-09-20T22:22:14.552 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: Standby daemons: 2024-09-20T22:22:14.552 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: 2024-09-20T22:22:14.552 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: [mds.foofs.smithi194.lkroso{-1:24305} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.194:6832/1702667629,v1:172.21.15.194:6833/1702667629] compat {c=[1],r=[1],i=[7ff]}] 2024-09-20T22:22:14.552 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: mon.smithi037@-1(???).osd e72 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-20T22:22:14.552 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: mon.smithi037@-1(???).osd e72 crush map has features 432629239337189376, adjusting msgr requires 2024-09-20T22:22:14.552 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: mon.smithi037@-1(???).osd e72 crush map has features 432629239337189376, adjusting msgr requires 2024-09-20T22:22:14.553 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: mon.smithi037@-1(???).osd e72 crush map has features 432629239337189376, adjusting msgr requires 2024-09-20T22:22:14.553 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 ceph-mon[65640]: mon.smithi037@-1(???).paxosservice(auth 1..29) refresh upgraded, format 0 -> 3 2024-09-20T22:22:14.553 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 bash[65621]: c0229eceb527050ba8644ebd6d5faff53bf9e2c05aec6ccdd04a9821ec899cf7 2024-09-20T22:22:14.553 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:14 smithi037 systemd[1]: Started Ceph mon.smithi037 for f9a6357a-779c-11ef-baf5-efdc52797490. 2024-09-20T22:22:14.836 INFO:teuthology.orchestra.run.smithi037.stdout: 5 123920 11.95 MB/sec execute 152 sec latency 35.168 ms 2024-09-20T22:22:15.295 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:15 smithi037 ceph-mon[65640]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mon metadata", "id": "smithi037"}]: dispatch 2024-09-20T22:22:15.295 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:15 smithi037 ceph-mon[65640]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mon metadata", "id": "smithi194"}]: dispatch 2024-09-20T22:22:15.296 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:15 smithi037 ceph-mon[65640]: pgmap v29: 97 pgs: 97 active+clean; 338 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 8.2 MiB/s wr, 769 op/s 2024-09-20T22:22:15.296 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:15 smithi037 ceph-mon[65640]: mon.smithi037 calling monitor election 2024-09-20T22:22:15.296 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:15 smithi037 ceph-mon[65640]: mon.smithi037 is new leader, mons smithi037,smithi194 in quorum (ranks 0,1) 2024-09-20T22:22:15.296 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:15 smithi037 ceph-mon[65640]: monmap epoch 2 2024-09-20T22:22:15.296 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:15 smithi037 ceph-mon[65640]: fsid f9a6357a-779c-11ef-baf5-efdc52797490 2024-09-20T22:22:15.296 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:15 smithi037 ceph-mon[65640]: last_changed 2024-09-20T22:11:58.559736+0000 2024-09-20T22:22:15.296 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:15 smithi037 ceph-mon[65640]: created 2024-09-20T22:09:36.885927+0000 2024-09-20T22:22:15.296 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:15 smithi037 ceph-mon[65640]: min_mon_release 17 (quincy) 2024-09-20T22:22:15.296 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:15 smithi037 ceph-mon[65640]: election_strategy: 1 2024-09-20T22:22:15.296 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:15 smithi037 ceph-mon[65640]: 0: [v2:172.21.15.37:3300/0,v1:172.21.15.37:6789/0] mon.smithi037 2024-09-20T22:22:15.296 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:15 smithi037 ceph-mon[65640]: 1: [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] mon.smithi194 2024-09-20T22:22:15.296 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:15 smithi037 ceph-mon[65640]: fsmap foofs:1 {0=foofs.smithi037.sdtedi=up:active} 1 up:standby 2024-09-20T22:22:15.296 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:15 smithi037 ceph-mon[65640]: osdmap e72: 8 total, 8 up, 8 in 2024-09-20T22:22:15.296 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:15 smithi037 ceph-mon[65640]: mgrmap e36: smithi037.lykblh(active, since 54s), standbys: smithi194.bkcglt 2024-09-20T22:22:15.296 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:15 smithi037 ceph-mon[65640]: overall HEALTH_OK 2024-09-20T22:22:15.297 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:15 smithi037 ceph-mon[65640]: mgrmap e37: smithi037.lykblh(active, since 54s), standbys: smithi194.bkcglt 2024-09-20T22:22:15.297 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:15 smithi037 ceph-mon[65640]: from='mgr.14780 ' entity='' 2024-09-20T22:22:15.521 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:15 smithi194 ceph-mon[25957]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mon metadata", "id": "smithi037"}]: dispatch 2024-09-20T22:22:15.521 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:15 smithi194 ceph-mon[25957]: from='mgr.14780 172.21.15.37:0/4191975361' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mon metadata", "id": "smithi194"}]: dispatch 2024-09-20T22:22:15.521 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:15 smithi194 ceph-mon[25957]: pgmap v29: 97 pgs: 97 active+clean; 338 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 8.2 MiB/s wr, 769 op/s 2024-09-20T22:22:15.522 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:15 smithi194 ceph-mon[25957]: mon.smithi037 calling monitor election 2024-09-20T22:22:15.522 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:15 smithi194 ceph-mon[25957]: mon.smithi037 is new leader, mons smithi037,smithi194 in quorum (ranks 0,1) 2024-09-20T22:22:15.522 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:15 smithi194 ceph-mon[25957]: monmap epoch 2 2024-09-20T22:22:15.522 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:15 smithi194 ceph-mon[25957]: fsid f9a6357a-779c-11ef-baf5-efdc52797490 2024-09-20T22:22:15.522 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:15 smithi194 ceph-mon[25957]: last_changed 2024-09-20T22:11:58.559736+0000 2024-09-20T22:22:15.522 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:15 smithi194 ceph-mon[25957]: created 2024-09-20T22:09:36.885927+0000 2024-09-20T22:22:15.522 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:15 smithi194 ceph-mon[25957]: min_mon_release 17 (quincy) 2024-09-20T22:22:15.522 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:15 smithi194 ceph-mon[25957]: election_strategy: 1 2024-09-20T22:22:15.522 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:15 smithi194 ceph-mon[25957]: 0: [v2:172.21.15.37:3300/0,v1:172.21.15.37:6789/0] mon.smithi037 2024-09-20T22:22:15.522 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:15 smithi194 ceph-mon[25957]: 1: [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] mon.smithi194 2024-09-20T22:22:15.522 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:15 smithi194 ceph-mon[25957]: fsmap foofs:1 {0=foofs.smithi037.sdtedi=up:active} 1 up:standby 2024-09-20T22:22:15.522 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:15 smithi194 ceph-mon[25957]: osdmap e72: 8 total, 8 up, 8 in 2024-09-20T22:22:15.522 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:15 smithi194 ceph-mon[25957]: mgrmap e36: smithi037.lykblh(active, since 54s), standbys: smithi194.bkcglt 2024-09-20T22:22:15.523 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:15 smithi194 ceph-mon[25957]: overall HEALTH_OK 2024-09-20T22:22:15.523 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:15 smithi194 ceph-mon[25957]: mgrmap e37: smithi037.lykblh(active, since 54s), standbys: smithi194.bkcglt 2024-09-20T22:22:15.523 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:15 smithi194 ceph-mon[25957]: from='mgr.14780 ' entity='' 2024-09-20T22:22:15.836 INFO:teuthology.orchestra.run.smithi037.stdout: 5 124248 11.93 MB/sec execute 153 sec latency 53.394 ms 2024-09-20T22:22:16.836 INFO:teuthology.orchestra.run.smithi037.stdout: 5 124574 11.92 MB/sec execute 154 sec latency 65.339 ms 2024-09-20T22:22:17.099 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:17 smithi037 ceph-mon[65640]: mgrmap e38: smithi037.lykblh(active, since 55s), standbys: smithi194.bkcglt 2024-09-20T22:22:17.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:17 smithi194 ceph-mon[25957]: mgrmap e38: smithi037.lykblh(active, since 55s), standbys: smithi194.bkcglt 2024-09-20T22:22:17.836 INFO:teuthology.orchestra.run.smithi037.stdout: 5 124750 11.88 MB/sec execute 155 sec latency 42.388 ms 2024-09-20T22:22:18.836 INFO:teuthology.orchestra.run.smithi037.stdout: 5 124977 11.84 MB/sec execute 156 sec latency 53.725 ms 2024-09-20T22:22:19.837 INFO:teuthology.orchestra.run.smithi037.stdout: 5 125276 11.78 MB/sec execute 157 sec latency 116.094 ms 2024-09-20T22:22:20.837 INFO:teuthology.orchestra.run.smithi037.stdout: 5 125576 11.73 MB/sec execute 158 sec latency 82.046 ms 2024-09-20T22:22:21.183 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:21 smithi037 ceph-mon[65640]: Standby manager daemon smithi194.bkcglt restarted 2024-09-20T22:22:21.183 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:21 smithi037 ceph-mon[65640]: Standby manager daemon smithi194.bkcglt started 2024-09-20T22:22:21.184 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:21 smithi037 ceph-mon[65640]: from='mgr.? 172.21.15.194:0/205698562' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi194.bkcglt/crt"}]: dispatch 2024-09-20T22:22:21.184 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:21 smithi037 ceph-mon[65640]: from='mgr.? 172.21.15.194:0/205698562' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-20T22:22:21.184 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:21 smithi037 ceph-mon[65640]: from='mgr.? 172.21.15.194:0/205698562' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi194.bkcglt/key"}]: dispatch 2024-09-20T22:22:21.184 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:21 smithi037 ceph-mon[65640]: from='mgr.? 172.21.15.194:0/205698562' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-20T22:22:21.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:21 smithi194 ceph-mon[25957]: Standby manager daemon smithi194.bkcglt restarted 2024-09-20T22:22:21.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:21 smithi194 ceph-mon[25957]: Standby manager daemon smithi194.bkcglt started 2024-09-20T22:22:21.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:21 smithi194 ceph-mon[25957]: from='mgr.? 172.21.15.194:0/205698562' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi194.bkcglt/crt"}]: dispatch 2024-09-20T22:22:21.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:21 smithi194 ceph-mon[25957]: from='mgr.? 172.21.15.194:0/205698562' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-20T22:22:21.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:21 smithi194 ceph-mon[25957]: from='mgr.? 172.21.15.194:0/205698562' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi194.bkcglt/key"}]: dispatch 2024-09-20T22:22:21.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:21 smithi194 ceph-mon[25957]: from='mgr.? 172.21.15.194:0/205698562' entity='mgr.smithi194.bkcglt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-20T22:22:21.837 INFO:teuthology.orchestra.run.smithi037.stdout: 5 125817 11.70 MB/sec execute 159 sec latency 41.297 ms 2024-09-20T22:22:22.204 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:22 smithi037 ceph-mon[65640]: mgrmap e39: smithi037.lykblh(active, since 61s), standbys: smithi194.bkcglt 2024-09-20T22:22:22.204 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:22 smithi037 ceph-mon[65640]: Active manager daemon smithi037.lykblh restarted 2024-09-20T22:22:22.204 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:22 smithi037 ceph-mon[65640]: Activating manager daemon smithi037.lykblh 2024-09-20T22:22:22.204 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:22 smithi037 ceph-mon[65640]: osdmap e73: 8 total, 8 up, 8 in 2024-09-20T22:22:22.204 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:22 smithi037 ceph-mon[65640]: mgrmap e40: smithi037.lykblh(active, starting, since 0.00752548s), standbys: smithi194.bkcglt 2024-09-20T22:22:22.204 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:22 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mon metadata", "id": "smithi037"}]: dispatch 2024-09-20T22:22:22.205 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:22 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mon metadata", "id": "smithi194"}]: dispatch 2024-09-20T22:22:22.205 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:22 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi037.sdtedi"}]: dispatch 2024-09-20T22:22:22.205 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:22 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi194.lkroso"}]: dispatch 2024-09-20T22:22:22.205 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:22 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mgr metadata", "who": "smithi037.lykblh", "id": "smithi037.lykblh"}]: dispatch 2024-09-20T22:22:22.205 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:22 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mgr metadata", "who": "smithi194.bkcglt", "id": "smithi194.bkcglt"}]: dispatch 2024-09-20T22:22:22.205 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:22 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-20T22:22:22.205 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:22 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-20T22:22:22.205 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:22 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-20T22:22:22.205 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:22 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-20T22:22:22.205 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:22 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-20T22:22:22.205 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:22 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-20T22:22:22.205 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:22 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-20T22:22:22.205 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:22 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-20T22:22:22.206 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:22 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-20T22:22:22.206 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:22 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-20T22:22:22.206 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:22 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-20T22:22:22.206 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:22 smithi037 ceph-mon[65640]: Manager daemon smithi037.lykblh is now available 2024-09-20T22:22:22.206 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:22 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:22:22.206 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:22 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:22:22.206 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:22 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi037.lykblh/mirror_snapshot_schedule"}]: dispatch 2024-09-20T22:22:22.206 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:22 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi037.lykblh/trash_purge_schedule"}]: dispatch 2024-09-20T22:22:22.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:22 smithi194 ceph-mon[25957]: mgrmap e39: smithi037.lykblh(active, since 61s), standbys: smithi194.bkcglt 2024-09-20T22:22:22.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:22 smithi194 ceph-mon[25957]: Active manager daemon smithi037.lykblh restarted 2024-09-20T22:22:22.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:22 smithi194 ceph-mon[25957]: Activating manager daemon smithi037.lykblh 2024-09-20T22:22:22.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:22 smithi194 ceph-mon[25957]: osdmap e73: 8 total, 8 up, 8 in 2024-09-20T22:22:22.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:22 smithi194 ceph-mon[25957]: mgrmap e40: smithi037.lykblh(active, starting, since 0.00752548s), standbys: smithi194.bkcglt 2024-09-20T22:22:22.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:22 smithi194 ceph-mon[25957]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mon metadata", "id": "smithi037"}]: dispatch 2024-09-20T22:22:22.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:22 smithi194 ceph-mon[25957]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mon metadata", "id": "smithi194"}]: dispatch 2024-09-20T22:22:22.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:22 smithi194 ceph-mon[25957]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi037.sdtedi"}]: dispatch 2024-09-20T22:22:22.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:22 smithi194 ceph-mon[25957]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi194.lkroso"}]: dispatch 2024-09-20T22:22:22.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:22 smithi194 ceph-mon[25957]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mgr metadata", "who": "smithi037.lykblh", "id": "smithi037.lykblh"}]: dispatch 2024-09-20T22:22:22.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:22 smithi194 ceph-mon[25957]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mgr metadata", "who": "smithi194.bkcglt", "id": "smithi194.bkcglt"}]: dispatch 2024-09-20T22:22:22.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:22 smithi194 ceph-mon[25957]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-20T22:22:22.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:22 smithi194 ceph-mon[25957]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-20T22:22:22.346 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:22 smithi194 ceph-mon[25957]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-20T22:22:22.346 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:22 smithi194 ceph-mon[25957]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-20T22:22:22.346 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:22 smithi194 ceph-mon[25957]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-20T22:22:22.346 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:22 smithi194 ceph-mon[25957]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-20T22:22:22.346 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:22 smithi194 ceph-mon[25957]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-20T22:22:22.346 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:22 smithi194 ceph-mon[25957]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-20T22:22:22.346 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:22 smithi194 ceph-mon[25957]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-20T22:22:22.346 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:22 smithi194 ceph-mon[25957]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-20T22:22:22.346 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:22 smithi194 ceph-mon[25957]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-20T22:22:22.346 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:22 smithi194 ceph-mon[25957]: Manager daemon smithi037.lykblh is now available 2024-09-20T22:22:22.346 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:22 smithi194 ceph-mon[25957]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:22:22.346 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:22 smithi194 ceph-mon[25957]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:22:22.346 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:22 smithi194 ceph-mon[25957]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi037.lykblh/mirror_snapshot_schedule"}]: dispatch 2024-09-20T22:22:22.347 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:22 smithi194 ceph-mon[25957]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi037.lykblh/trash_purge_schedule"}]: dispatch 2024-09-20T22:22:22.837 INFO:teuthology.orchestra.run.smithi037.stdout: 5 126198 11.73 MB/sec execute 160 sec latency 48.334 ms 2024-09-20T22:22:23.837 INFO:teuthology.orchestra.run.smithi037.stdout: 5 126561 11.69 MB/sec execute 161 sec latency 66.072 ms 2024-09-20T22:22:24.007 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:23 smithi037 ceph-mon[65640]: mgrmap e41: smithi037.lykblh(active, since 1.01083s), standbys: smithi194.bkcglt 2024-09-20T22:22:24.007 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:23 smithi037 ceph-mon[65640]: pgmap v3: 97 pgs: 97 active+clean; 329 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-20T22:22:24.077 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:23 smithi194 ceph-mon[25957]: mgrmap e41: smithi037.lykblh(active, since 1.01083s), standbys: smithi194.bkcglt 2024-09-20T22:22:24.078 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:23 smithi194 ceph-mon[25957]: pgmap v3: 97 pgs: 97 active+clean; 329 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-20T22:22:24.837 INFO:teuthology.orchestra.run.smithi037.stdout: 5 127112 11.71 MB/sec execute 162 sec latency 30.998 ms 2024-09-20T22:22:25.018 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:24 smithi037 ceph-mon[65640]: [20/Sep/2024:22:22:23] ENGINE Bus STARTING 2024-09-20T22:22:25.018 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:24 smithi037 ceph-mon[65640]: pgmap v4: 97 pgs: 97 active+clean; 329 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-20T22:22:25.018 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:24 smithi037 ceph-mon[65640]: mgrmap e42: smithi037.lykblh(active, since 2s), standbys: smithi194.bkcglt 2024-09-20T22:22:25.018 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:24 smithi037 ceph-mon[65640]: [20/Sep/2024:22:22:23] ENGINE Serving on http://172.21.15.37:8765 2024-09-20T22:22:25.018 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:24 smithi037 ceph-mon[65640]: [20/Sep/2024:22:22:24] ENGINE Serving on https://172.21.15.37:7150 2024-09-20T22:22:25.018 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:24 smithi037 ceph-mon[65640]: [20/Sep/2024:22:22:24] ENGINE Bus STARTED 2024-09-20T22:22:25.018 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:24 smithi037 ceph-mon[65640]: [20/Sep/2024:22:22:24] ENGINE Error in HTTPServer.serve 2024-09-20T22:22:25.018 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: Traceback (most recent call last): 2024-09-20T22:22:25.018 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-20T22:22:25.018 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: self._connections.run(self.expiration_interval) 2024-09-20T22:22:25.018 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-20T22:22:25.018 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: self._run(expiration_interval) 2024-09-20T22:22:25.019 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-20T22:22:25.019 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-20T22:22:25.019 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-20T22:22:25.019 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-20T22:22:25.019 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-20T22:22:25.019 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: s = self.context.wrap_socket( 2024-09-20T22:22:25.019 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-20T22:22:25.019 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: return self.sslsocket_class._create( 2024-09-20T22:22:25.019 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-20T22:22:25.019 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: self.do_handshake() 2024-09-20T22:22:25.019 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-20T22:22:25.019 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: self._sslobj.do_handshake() 2024-09-20T22:22:25.019 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-20T22:22:25.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:24 smithi194 ceph-mon[25957]: [20/Sep/2024:22:22:23] ENGINE Bus STARTING 2024-09-20T22:22:25.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:24 smithi194 ceph-mon[25957]: pgmap v4: 97 pgs: 97 active+clean; 329 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-20T22:22:25.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:24 smithi194 ceph-mon[25957]: mgrmap e42: smithi037.lykblh(active, since 2s), standbys: smithi194.bkcglt 2024-09-20T22:22:25.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:24 smithi194 ceph-mon[25957]: [20/Sep/2024:22:22:23] ENGINE Serving on http://172.21.15.37:8765 2024-09-20T22:22:25.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:24 smithi194 ceph-mon[25957]: [20/Sep/2024:22:22:24] ENGINE Serving on https://172.21.15.37:7150 2024-09-20T22:22:25.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:24 smithi194 ceph-mon[25957]: [20/Sep/2024:22:22:24] ENGINE Bus STARTED 2024-09-20T22:22:25.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:24 smithi194 ceph-mon[25957]: [20/Sep/2024:22:22:24] ENGINE Error in HTTPServer.serve 2024-09-20T22:22:25.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: Traceback (most recent call last): 2024-09-20T22:22:25.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-20T22:22:25.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: self._connections.run(self.expiration_interval) 2024-09-20T22:22:25.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-20T22:22:25.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: self._run(expiration_interval) 2024-09-20T22:22:25.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-20T22:22:25.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-20T22:22:25.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-20T22:22:25.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-20T22:22:25.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-20T22:22:25.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: s = self.context.wrap_socket( 2024-09-20T22:22:25.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-20T22:22:25.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: return self.sslsocket_class._create( 2024-09-20T22:22:25.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-20T22:22:25.096 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: self.do_handshake() 2024-09-20T22:22:25.096 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-20T22:22:25.096 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: self._sslobj.do_handshake() 2024-09-20T22:22:25.096 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-20T22:22:25.838 INFO:teuthology.orchestra.run.smithi037.stdout: 5 127524 11.74 MB/sec execute 163 sec latency 25.658 ms 2024-09-20T22:22:26.803 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:26 smithi037 ceph-mon[65640]: pgmap v5: 97 pgs: 97 active+clean; 329 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-20T22:22:26.803 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:26 smithi037 ceph-mon[65640]: mgrmap e43: smithi037.lykblh(active, since 4s), standbys: smithi194.bkcglt 2024-09-20T22:22:26.803 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:26 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:26.805 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:26 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:26.837 INFO:teuthology.orchestra.run.smithi037.stdout: 5 127958 11.73 MB/sec execute 164 sec latency 85.222 ms 2024-09-20T22:22:27.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:26 smithi194 ceph-mon[25957]: pgmap v5: 97 pgs: 97 active+clean; 329 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-20T22:22:27.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:26 smithi194 ceph-mon[25957]: mgrmap e43: smithi037.lykblh(active, since 4s), standbys: smithi194.bkcglt 2024-09-20T22:22:27.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:26 smithi194 ceph-mon[25957]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:27.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:26 smithi194 ceph-mon[25957]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:27.838 INFO:teuthology.orchestra.run.smithi037.stdout: 5 128220 11.72 MB/sec execute 165 sec latency 57.948 ms 2024-09-20T22:22:28.333 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:28 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:28.333 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:28 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:28.398 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:28 smithi194 ceph-mon[25957]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:28.398 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:28 smithi194 ceph-mon[25957]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:28.838 INFO:teuthology.orchestra.run.smithi037.stdout: 5 128434 11.69 MB/sec execute 166 sec latency 43.611 ms 2024-09-20T22:22:29.120 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:29 smithi037 ceph-mon[65640]: pgmap v6: 97 pgs: 97 active+clean; 329 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-20T22:22:29.338 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:29 smithi194 ceph-mon[25957]: pgmap v6: 97 pgs: 97 active+clean; 329 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-20T22:22:29.838 INFO:teuthology.orchestra.run.smithi037.stdout: 5 128669 11.65 MB/sec execute 167 sec latency 91.334 ms 2024-09-20T22:22:30.838 INFO:teuthology.orchestra.run.smithi037.stdout: 5 128971 11.59 MB/sec execute 168 sec latency 70.429 ms 2024-09-20T22:22:31.189 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:31 smithi037 ceph-mon[65640]: pgmap v7: 97 pgs: 97 active+clean; 340 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 35 KiB/s rd, 4.5 MiB/s wr, 419 op/s 2024-09-20T22:22:31.189 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:31 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:31.190 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:31 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:31.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:31 smithi194 ceph-mon[25957]: pgmap v7: 97 pgs: 97 active+clean; 340 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 35 KiB/s rd, 4.5 MiB/s wr, 419 op/s 2024-09-20T22:22:31.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:31 smithi194 ceph-mon[25957]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:31.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:31 smithi194 ceph-mon[25957]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:31.838 INFO:teuthology.orchestra.run.smithi037.stdout: 5 129224 11.54 MB/sec execute 169 sec latency 51.115 ms 2024-09-20T22:22:32.487 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:32 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:32.487 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:32 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:32.488 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:32 smithi037 ceph-mon[65640]: pgmap v8: 97 pgs: 97 active+clean; 340 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 27 KiB/s rd, 3.5 MiB/s wr, 326 op/s 2024-09-20T22:22:32.488 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:32 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:32.488 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:32 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:32.488 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:32 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:22:32.488 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:32 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:22:32.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:32 smithi194 ceph-mon[25957]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:32.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:32 smithi194 ceph-mon[25957]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:32.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:32 smithi194 ceph-mon[25957]: pgmap v8: 97 pgs: 97 active+clean; 340 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 27 KiB/s rd, 3.5 MiB/s wr, 326 op/s 2024-09-20T22:22:32.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:32 smithi194 ceph-mon[25957]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:32.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:32 smithi194 ceph-mon[25957]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:32.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:32 smithi194 ceph-mon[25957]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:22:32.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:32 smithi194 ceph-mon[25957]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:22:32.838 INFO:teuthology.orchestra.run.smithi037.stdout: 5 129553 11.53 MB/sec execute 170 sec latency 44.490 ms 2024-09-20T22:22:33.503 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:33 smithi037 ceph-mon[65640]: Updating smithi037:/etc/ceph/ceph.conf 2024-09-20T22:22:33.503 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:33 smithi037 ceph-mon[65640]: Updating smithi194:/etc/ceph/ceph.conf 2024-09-20T22:22:33.503 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:33 smithi037 ceph-mon[65640]: Updating smithi194:/var/lib/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/config/ceph.conf 2024-09-20T22:22:33.503 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:33 smithi037 ceph-mon[65640]: Updating smithi037:/var/lib/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/config/ceph.conf 2024-09-20T22:22:33.503 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:33 smithi037 ceph-mon[65640]: Updating smithi194:/etc/ceph/ceph.client.admin.keyring 2024-09-20T22:22:33.503 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:33 smithi037 ceph-mon[65640]: Updating smithi037:/etc/ceph/ceph.client.admin.keyring 2024-09-20T22:22:33.503 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:33 smithi037 ceph-mon[65640]: Updating smithi194:/var/lib/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/config/ceph.client.admin.keyring 2024-09-20T22:22:33.503 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:33 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:33.504 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:33 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:33.838 INFO:teuthology.orchestra.run.smithi037.stdout: 5 130009 11.56 MB/sec execute 171 sec latency 35.481 ms 2024-09-20T22:22:33.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:33 smithi194 ceph-mon[25957]: Updating smithi037:/etc/ceph/ceph.conf 2024-09-20T22:22:33.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:33 smithi194 ceph-mon[25957]: Updating smithi194:/etc/ceph/ceph.conf 2024-09-20T22:22:33.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:33 smithi194 ceph-mon[25957]: Updating smithi194:/var/lib/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/config/ceph.conf 2024-09-20T22:22:33.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:33 smithi194 ceph-mon[25957]: Updating smithi037:/var/lib/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/config/ceph.conf 2024-09-20T22:22:33.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:33 smithi194 ceph-mon[25957]: Updating smithi194:/etc/ceph/ceph.client.admin.keyring 2024-09-20T22:22:33.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:33 smithi194 ceph-mon[25957]: Updating smithi037:/etc/ceph/ceph.client.admin.keyring 2024-09-20T22:22:33.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:33 smithi194 ceph-mon[25957]: Updating smithi194:/var/lib/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/config/ceph.client.admin.keyring 2024-09-20T22:22:33.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:33 smithi194 ceph-mon[25957]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:33.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:33 smithi194 ceph-mon[25957]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:34.268 INFO:teuthology.orchestra.run.smithi037.stdout:true 2024-09-20T22:22:34.520 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:34 smithi037 ceph-mon[65640]: Updating smithi037:/var/lib/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/config/ceph.client.admin.keyring 2024-09-20T22:22:34.521 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:34 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:34.521 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:34 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:34.521 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:34 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:34.521 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:34 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:34.521 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:34 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:22:34.521 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:34 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:22:34.521 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:34 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-20T22:22:34.521 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:34 smithi037 ceph-mon[65640]: pgmap v9: 97 pgs: 97 active+clean; 333 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 26 KiB/s rd, 4.7 MiB/s wr, 466 op/s 2024-09-20T22:22:34.838 INFO:teuthology.orchestra.run.smithi037.stdout: 5 130658 11.58 MB/sec execute 172 sec latency 25.007 ms 2024-09-20T22:22:34.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:34 smithi194 ceph-mon[25957]: Updating smithi037:/var/lib/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/config/ceph.client.admin.keyring 2024-09-20T22:22:34.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:34 smithi194 ceph-mon[25957]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:34.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:34 smithi194 ceph-mon[25957]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:34.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:34 smithi194 ceph-mon[25957]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:34.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:34 smithi194 ceph-mon[25957]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:34.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:34 smithi194 ceph-mon[25957]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:22:34.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:34 smithi194 ceph-mon[25957]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:22:34.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:34 smithi194 ceph-mon[25957]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-20T22:22:34.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:34 smithi194 ceph-mon[25957]: pgmap v9: 97 pgs: 97 active+clean; 333 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 26 KiB/s rd, 4.7 MiB/s wr, 466 op/s 2024-09-20T22:22:35.374 INFO:teuthology.orchestra.run.smithi037.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T22:22:35.375 INFO:teuthology.orchestra.run.smithi037.stdout:alertmanager.smithi037 smithi037 *:9093,9094 running (2m) 4s ago 11m 27.9M - 0.25.0 c8568f914cd2 d1f99b3cd707 2024-09-20T22:22:35.375 INFO:teuthology.orchestra.run.smithi037.stdout:crash.smithi037 smithi037 running (11m) 4s ago 11m 6400k - 17.2.0 e1d6a67b021e 88301022bdc8 2024-09-20T22:22:35.375 INFO:teuthology.orchestra.run.smithi037.stdout:crash.smithi194 smithi194 running (10m) 9s ago 10m 6405k - 17.2.0 e1d6a67b021e b553601d7ca5 2024-09-20T22:22:35.375 INFO:teuthology.orchestra.run.smithi037.stdout:grafana.smithi037 smithi037 *:3000 running (2m) 4s ago 11m 51.7M - dad864ee21e9 2306f3c2da19 2024-09-20T22:22:35.375 INFO:teuthology.orchestra.run.smithi037.stdout:mds.foofs.smithi037.sdtedi smithi037 running (5m) 4s ago 5m 37.2M - 17.2.0 e1d6a67b021e 0a7898d25b8d 2024-09-20T22:22:35.375 INFO:teuthology.orchestra.run.smithi037.stdout:mds.foofs.smithi194.lkroso smithi194 running (5m) 9s ago 5m 15.3M - 17.2.0 e1d6a67b021e 17736de30008 2024-09-20T22:22:35.375 INFO:teuthology.orchestra.run.smithi037.stdout:mgr.smithi037.lykblh smithi037 *:8443,9283,8765 running (99s) 4s ago 12m 475M - 19.1.1-367-g9e2ac70b cc9251b744a3 181e46fd9ba6 2024-09-20T22:22:35.375 INFO:teuthology.orchestra.run.smithi037.stdout:mgr.smithi194.bkcglt smithi194 *:8443,9283,8765 running (40s) 9s ago 10m 437M - 19.1.1-367-g9e2ac70b cc9251b744a3 1fa610c97517 2024-09-20T22:22:35.375 INFO:teuthology.orchestra.run.smithi037.stdout:mon.smithi037 smithi037 running (21s) 4s ago 12m 30.9M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 c0229eceb527 2024-09-20T22:22:35.375 INFO:teuthology.orchestra.run.smithi037.stdout:mon.smithi194 smithi194 running (10m) 9s ago 10m 48.2M 2048M 17.2.0 e1d6a67b021e 0b8814ae6c66 2024-09-20T22:22:35.375 INFO:teuthology.orchestra.run.smithi037.stdout:nfs.foo.0.0.smithi037.spabdc smithi037 *:2049 running (5m) 4s ago 5m 265M - 4.0 e1d6a67b021e 858ff9d558a2 2024-09-20T22:22:35.375 INFO:teuthology.orchestra.run.smithi037.stdout:nfs.foo.1.0.smithi194.qzjyej smithi194 *:2049 running (5m) 9s ago 5m 89.6M - 4.0 e1d6a67b021e c07fe0eb7c75 2024-09-20T22:22:35.375 INFO:teuthology.orchestra.run.smithi037.stdout:node-exporter.smithi037 smithi037 *:9100 running (2m) 4s ago 11m 9915k - 1.5.0 0da6a335fe13 433e6aaf6137 2024-09-20T22:22:35.375 INFO:teuthology.orchestra.run.smithi037.stdout:node-exporter.smithi194 smithi194 *:9100 running (108s) 9s ago 10m 12.0M - 1.5.0 0da6a335fe13 1d30bb810ad1 2024-09-20T22:22:35.376 INFO:teuthology.orchestra.run.smithi037.stdout:osd.0 smithi037 running (9m) 4s ago 9m 463M 1937M 17.2.0 e1d6a67b021e e54e84fd5030 2024-09-20T22:22:35.376 INFO:teuthology.orchestra.run.smithi037.stdout:osd.1 smithi037 running (9m) 4s ago 9m 403M 1937M 17.2.0 e1d6a67b021e 09647c133032 2024-09-20T22:22:35.376 INFO:teuthology.orchestra.run.smithi037.stdout:osd.2 smithi037 running (8m) 4s ago 8m 290M 1937M 17.2.0 e1d6a67b021e 6c15b1175256 2024-09-20T22:22:35.376 INFO:teuthology.orchestra.run.smithi037.stdout:osd.3 smithi037 running (8m) 4s ago 8m 426M 1937M 17.2.0 e1d6a67b021e 3a22952c6475 2024-09-20T22:22:35.376 INFO:teuthology.orchestra.run.smithi037.stdout:osd.4 smithi194 running (8m) 9s ago 8m 445M 2704M 17.2.0 e1d6a67b021e fa107be8915c 2024-09-20T22:22:35.376 INFO:teuthology.orchestra.run.smithi037.stdout:osd.5 smithi194 running (7m) 9s ago 7m 490M 2704M 17.2.0 e1d6a67b021e 290b21cbd131 2024-09-20T22:22:35.376 INFO:teuthology.orchestra.run.smithi037.stdout:osd.6 smithi194 running (7m) 9s ago 7m 414M 2704M 17.2.0 e1d6a67b021e 27f0720639ed 2024-09-20T22:22:35.376 INFO:teuthology.orchestra.run.smithi037.stdout:osd.7 smithi194 running (6m) 9s ago 6m 573M 2704M 17.2.0 e1d6a67b021e 62d163f4fd92 2024-09-20T22:22:35.376 INFO:teuthology.orchestra.run.smithi037.stdout:prometheus.smithi037 smithi037 *:9095 running (46s) 4s ago 10m 47.2M - 2.43.0 a07b618ecd1d 05b826dde2ff 2024-09-20T22:22:35.545 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:35 smithi037 ceph-mon[65640]: from='client.34128 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:22:35.545 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:35 smithi037 ceph-mon[65640]: from='client.24661 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:22:35.545 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:35 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:35.545 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:35 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-20T22:22:35.545 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:35 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-20T22:22:35.545 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:35 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:22:35.838 INFO:teuthology.orchestra.run.smithi037.stdout: 5 131136 11.62 MB/sec execute 173 sec latency 32.071 ms 2024-09-20T22:22:35.840 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:35 smithi194 ceph-mon[25957]: from='client.34128 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:22:35.840 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:35 smithi194 ceph-mon[25957]: from='client.24661 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:22:35.840 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:35 smithi194 ceph-mon[25957]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:35.840 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:35 smithi194 ceph-mon[25957]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-20T22:22:35.840 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:35 smithi194 ceph-mon[25957]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-20T22:22:35.840 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:35 smithi194 ceph-mon[25957]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:22:35.999 INFO:teuthology.orchestra.run.smithi037.stdout:{ 2024-09-20T22:22:36.000 INFO:teuthology.orchestra.run.smithi037.stdout: "mon": { 2024-09-20T22:22:36.000 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-20T22:22:36.000 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 1 2024-09-20T22:22:36.000 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:22:36.000 INFO:teuthology.orchestra.run.smithi037.stdout: "mgr": { 2024-09-20T22:22:36.000 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 2 2024-09-20T22:22:36.000 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:22:36.000 INFO:teuthology.orchestra.run.smithi037.stdout: "osd": { 2024-09-20T22:22:36.000 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-20T22:22:36.000 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:22:36.000 INFO:teuthology.orchestra.run.smithi037.stdout: "mds": { 2024-09-20T22:22:36.000 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T22:22:36.000 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:22:36.001 INFO:teuthology.orchestra.run.smithi037.stdout: "overall": { 2024-09-20T22:22:36.001 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 11, 2024-09-20T22:22:36.001 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 3 2024-09-20T22:22:36.001 INFO:teuthology.orchestra.run.smithi037.stdout: } 2024-09-20T22:22:36.001 INFO:teuthology.orchestra.run.smithi037.stdout:} 2024-09-20T22:22:36.565 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:36 smithi037 ceph-mon[65640]: from='client.24665 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:22:36.565 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:36 smithi037 ceph-mon[65640]: Upgrade: Updating mon.smithi194 2024-09-20T22:22:36.565 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:36 smithi037 ceph-mon[65640]: Deploying daemon mon.smithi194 on smithi194 2024-09-20T22:22:36.566 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:36 smithi037 ceph-mon[65640]: pgmap v10: 97 pgs: 97 active+clean; 333 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 24 KiB/s rd, 4.3 MiB/s wr, 427 op/s 2024-09-20T22:22:36.566 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:36 smithi037 ceph-mon[65640]: from='client.? 172.21.15.37:0/966301405' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:22:36.589 INFO:teuthology.orchestra.run.smithi037.stdout:{ 2024-09-20T22:22:36.590 INFO:teuthology.orchestra.run.smithi037.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5", 2024-09-20T22:22:36.590 INFO:teuthology.orchestra.run.smithi037.stdout: "in_progress": true, 2024-09-20T22:22:36.590 INFO:teuthology.orchestra.run.smithi037.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-20T22:22:36.590 INFO:teuthology.orchestra.run.smithi037.stdout: "services_complete": [ 2024-09-20T22:22:36.590 INFO:teuthology.orchestra.run.smithi037.stdout: "mgr" 2024-09-20T22:22:36.590 INFO:teuthology.orchestra.run.smithi037.stdout: ], 2024-09-20T22:22:36.590 INFO:teuthology.orchestra.run.smithi037.stdout: "progress": "3/23 daemons upgraded", 2024-09-20T22:22:36.590 INFO:teuthology.orchestra.run.smithi037.stdout: "message": "Currently upgrading mon daemons", 2024-09-20T22:22:36.590 INFO:teuthology.orchestra.run.smithi037.stdout: "is_paused": false 2024-09-20T22:22:36.590 INFO:teuthology.orchestra.run.smithi037.stdout:} 2024-09-20T22:22:36.838 INFO:teuthology.orchestra.run.smithi037.stdout: 5 131481 11.59 MB/sec execute 174 sec latency 49.627 ms 2024-09-20T22:22:36.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:36 smithi194 ceph-mon[25957]: from='client.24665 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:22:36.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:36 smithi194 ceph-mon[25957]: Upgrade: Updating mon.smithi194 2024-09-20T22:22:36.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:36 smithi194 ceph-mon[25957]: Deploying daemon mon.smithi194 on smithi194 2024-09-20T22:22:36.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:36 smithi194 ceph-mon[25957]: pgmap v10: 97 pgs: 97 active+clean; 333 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 24 KiB/s rd, 4.3 MiB/s wr, 427 op/s 2024-09-20T22:22:36.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:36 smithi194 ceph-mon[25957]: from='client.? 172.21.15.37:0/966301405' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:22:37.092 INFO:teuthology.orchestra.run.smithi037.stdout:HEALTH_OK 2024-09-20T22:22:37.492 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:37 smithi194 systemd[1]: Stopping Ceph mon.smithi194 for f9a6357a-779c-11ef-baf5-efdc52797490... 2024-09-20T22:22:37.492 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:37 smithi194 ceph-mon[25957]: from='client.34144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:22:37.492 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:37 smithi194 ceph-mon[25957]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:22:37.581 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:37 smithi037 ceph-mon[65640]: from='client.34144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:22:37.581 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:37 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:22:37.582 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:37 smithi037 ceph-mon[65640]: from='client.? 172.21.15.37:0/1527461697' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T22:22:37.838 INFO:teuthology.orchestra.run.smithi037.stdout: 5 131760 11.60 MB/sec execute 175 sec latency 72.060 ms 2024-09-20T22:22:37.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:37 smithi194 ceph-mon[25957]: from='client.? 172.21.15.37:0/1527461697' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T22:22:37.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:37 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-mon-smithi194[25945]: 2024-09-20T22:22:37.561+0000 7f1fdf690700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi194 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-20T22:22:37.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:37 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-mon-smithi194[25945]: 2024-09-20T22:22:37.561+0000 7f1fdf690700 -1 mon.smithi194@1(peon) e2 *** Got Signal Terminated *** 2024-09-20T22:22:37.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:37 smithi194 podman[51576]: 2024-09-20 22:22:37.593608966 +0000 UTC m=+0.174279670 container died 0b8814ae6c6608926d23d2afe0cf35845092f7aec43c93ccc5c32d0913ffdde8 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-mon-smithi194, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , name=centos-stream, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, version=8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container) 2024-09-20T22:22:38.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:38 smithi194 podman[51576]: 2024-09-20 22:22:38.368329638 +0000 UTC m=+0.949000337 container cleanup 0b8814ae6c6608926d23d2afe0cf35845092f7aec43c93ccc5c32d0913ffdde8 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-mon-smithi194, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, version=8, vendor=Red Hat, Inc., name=centos-stream, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, vcs-type=git, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, io.openshift.expose-services=, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container) 2024-09-20T22:22:38.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:38 smithi194 bash[51576]: ceph-f9a6357a-779c-11ef-baf5-efdc52797490-mon-smithi194 2024-09-20T22:22:38.838 INFO:teuthology.orchestra.run.smithi037.stdout: 5 131962 11.57 MB/sec execute 176 sec latency 53.565 ms 2024-09-20T22:22:39.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:38 smithi194 podman[51587]: 2024-09-20 22:22:38.728285002 +0000 UTC m=+1.134023878 container remove 0b8814ae6c6608926d23d2afe0cf35845092f7aec43c93ccc5c32d0913ffdde8 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-mon-smithi194, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , release=754, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, distribution-scope=public, architecture=x86_64, vcs-type=git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD) 2024-09-20T22:22:39.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:38 smithi194 systemd[1]: ceph-f9a6357a-779c-11ef-baf5-efdc52797490@mon.smithi194.service: Deactivated successfully. 2024-09-20T22:22:39.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:38 smithi194 systemd[1]: Stopped Ceph mon.smithi194 for f9a6357a-779c-11ef-baf5-efdc52797490. 2024-09-20T22:22:39.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:38 smithi194 systemd[1]: ceph-f9a6357a-779c-11ef-baf5-efdc52797490@mon.smithi194.service: Consumed 6.992s CPU time. 2024-09-20T22:22:39.581 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 systemd[1]: Starting Ceph mon.smithi194 for f9a6357a-779c-11ef-baf5-efdc52797490... 2024-09-20T22:22:39.581 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 podman[51708]: 2024-09-20 22:22:39.292928512 +0000 UTC m=+0.018686261 image pull cc9251b744a3faafca7c3095f79bfff81873afe823f31a03a11e015a34abcdbd quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5 2024-09-20T22:22:39.836 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 podman[51708]: 2024-09-20 22:22:39.580330583 +0000 UTC m=+0.306088304 container create 97854807ba91a92fcb11515626eb160ee952ac236147761f68cbd8977b8382cd (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-mon-smithi194, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=squid-9e2ac70, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_BRANCH=HEAD) 2024-09-20T22:22:39.837 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 podman[51708]: 2024-09-20 22:22:39.808292564 +0000 UTC m=+0.534050286 container init 97854807ba91a92fcb11515626eb160ee952ac236147761f68cbd8977b8382cd (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-mon-smithi194, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, CEPH_POINT_RELEASE=, RELEASE=squid-9e2ac70, org.label-schema.build-date=20240731, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS) 2024-09-20T22:22:39.837 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 podman[51708]: 2024-09-20 22:22:39.811926921 +0000 UTC m=+0.537684640 container start 97854807ba91a92fcb11515626eb160ee952ac236147761f68cbd8977b8382cd (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-mon-smithi194, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=squid-9e2ac70, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.build-date=20240731) 2024-09-20T22:22:39.838 INFO:teuthology.orchestra.run.smithi037.stdout: 5 132202 11.53 MB/sec execute 177 sec latency 92.778 ms 2024-09-20T22:22:40.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: set uid:gid to 167:167 (ceph:ceph) 2024-09-20T22:22:40.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable), process ceph-mon, pid 2 2024-09-20T22:22:40.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: pidfile_write: ignore empty --pid-file 2024-09-20T22:22:40.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: load: jerasure load: lrc 2024-09-20T22:22:40.096 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: RocksDB version: 7.9.2 2024-09-20T22:22:40.096 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Git sha 0 2024-09-20T22:22:40.096 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Compile date 2024-09-19 18:20:48 2024-09-20T22:22:40.096 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: DB SUMMARY 2024-09-20T22:22:40.096 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: DB Session ID: 63W7YL6L2RQF3ARMDO8Z 2024-09-20T22:22:40.096 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: CURRENT file: CURRENT 2024-09-20T22:22:40.096 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: IDENTITY file: IDENTITY 2024-09-20T22:22:40.096 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: MANIFEST file: MANIFEST-000009 size: 905 Bytes 2024-09-20T22:22:40.096 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi194/store.db dir, Total Num: 1, files: 000024.sst 2024-09-20T22:22:40.096 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi194/store.db: 000022.log size: 5256803 ; 2024-09-20T22:22:40.096 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.error_if_exists: 0 2024-09-20T22:22:40.096 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.create_if_missing: 0 2024-09-20T22:22:40.096 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.paranoid_checks: 1 2024-09-20T22:22:40.097 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-20T22:22:40.097 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-20T22:22:40.097 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-20T22:22:40.097 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.env: 0x55e9631df940 2024-09-20T22:22:40.097 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.fs: PosixFileSystem 2024-09-20T22:22:40.097 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.info_log: 0x55e9648278c0 2024-09-20T22:22:40.097 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.max_file_opening_threads: 16 2024-09-20T22:22:40.097 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.statistics: (nil) 2024-09-20T22:22:40.097 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.use_fsync: 0 2024-09-20T22:22:40.097 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.max_log_file_size: 0 2024-09-20T22:22:40.097 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-20T22:22:40.097 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-20T22:22:40.097 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.keep_log_file_num: 1000 2024-09-20T22:22:40.097 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.recycle_log_file_num: 0 2024-09-20T22:22:40.097 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.allow_fallocate: 1 2024-09-20T22:22:40.098 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.allow_mmap_reads: 0 2024-09-20T22:22:40.098 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.allow_mmap_writes: 0 2024-09-20T22:22:40.098 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.use_direct_reads: 0 2024-09-20T22:22:40.098 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-20T22:22:40.098 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.create_missing_column_families: 0 2024-09-20T22:22:40.098 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.db_log_dir: 2024-09-20T22:22:40.098 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.wal_dir: 2024-09-20T22:22:40.098 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-20T22:22:40.099 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-20T22:22:40.099 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-20T22:22:40.099 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-20T22:22:40.100 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-20T22:22:40.100 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-20T22:22:40.100 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.advise_random_on_open: 1 2024-09-20T22:22:40.100 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.db_write_buffer_size: 0 2024-09-20T22:22:40.100 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.write_buffer_manager: 0x55e964836dc0 2024-09-20T22:22:40.100 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-20T22:22:40.100 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-20T22:22:40.100 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-20T22:22:40.100 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.rate_limiter: (nil) 2024-09-20T22:22:40.100 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-20T22:22:40.100 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.wal_recovery_mode: 2 2024-09-20T22:22:40.100 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.enable_thread_tracking: 0 2024-09-20T22:22:40.101 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.enable_pipelined_write: 0 2024-09-20T22:22:40.101 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.unordered_write: 0 2024-09-20T22:22:40.101 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-20T22:22:40.101 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-20T22:22:40.101 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-20T22:22:40.101 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-20T22:22:40.101 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.row_cache: None 2024-09-20T22:22:40.101 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.wal_filter: None 2024-09-20T22:22:40.101 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-20T22:22:40.101 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.allow_ingest_behind: 0 2024-09-20T22:22:40.101 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.two_write_queues: 0 2024-09-20T22:22:40.101 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.manual_wal_flush: 0 2024-09-20T22:22:40.101 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.wal_compression: 0 2024-09-20T22:22:40.101 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.atomic_flush: 0 2024-09-20T22:22:40.101 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-20T22:22:40.102 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-20T22:22:40.102 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-20T22:22:40.102 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.log_readahead_size: 0 2024-09-20T22:22:40.102 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-20T22:22:40.102 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.best_efforts_recovery: 0 2024-09-20T22:22:40.102 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-20T22:22:40.102 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-20T22:22:40.102 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.allow_data_in_errors: 0 2024-09-20T22:22:40.102 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.db_host_id: __hostname__ 2024-09-20T22:22:40.102 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-20T22:22:40.102 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.max_background_jobs: 2 2024-09-20T22:22:40.102 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.max_background_compactions: -1 2024-09-20T22:22:40.102 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.max_subcompactions: 1 2024-09-20T22:22:40.103 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-20T22:22:40.103 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-20T22:22:40.103 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-20T22:22:40.103 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.max_total_wal_size: 0 2024-09-20T22:22:40.103 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-20T22:22:40.103 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-20T22:22:40.103 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-20T22:22:40.103 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-20T22:22:40.103 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.max_open_files: -1 2024-09-20T22:22:40.103 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.bytes_per_sync: 0 2024-09-20T22:22:40.103 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-20T22:22:40.103 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-20T22:22:40.103 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.compaction_readahead_size: 0 2024-09-20T22:22:40.103 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.max_background_flushes: -1 2024-09-20T22:22:40.103 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Compression algorithms supported: 2024-09-20T22:22:40.104 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: kZSTD supported: 0 2024-09-20T22:22:40.104 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: kXpressCompression supported: 0 2024-09-20T22:22:40.104 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: kBZip2Compression supported: 0 2024-09-20T22:22:40.104 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-20T22:22:40.104 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: kLZ4Compression supported: 1 2024-09-20T22:22:40.104 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: kZlibCompression supported: 1 2024-09-20T22:22:40.104 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: kLZ4HCCompression supported: 1 2024-09-20T22:22:40.104 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: kSnappyCompression supported: 1 2024-09-20T22:22:40.104 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-20T22:22:40.104 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-20T22:22:40.104 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi194/store.db/MANIFEST-000009 2024-09-20T22:22:40.104 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-20T22:22:40.104 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-20T22:22:40.104 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.merge_operator: 2024-09-20T22:22:40.104 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.compaction_filter: None 2024-09-20T22:22:40.104 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.compaction_filter_factory: None 2024-09-20T22:22:40.105 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.sst_partitioner_factory: None 2024-09-20T22:22:40.105 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-20T22:22:40.105 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-20T22:22:40.105 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55e964827520) 2024-09-20T22:22:40.105 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: cache_index_and_filter_blocks: 1 2024-09-20T22:22:40.105 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-20T22:22:40.105 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-20T22:22:40.105 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: pin_top_level_index_and_filter: 1 2024-09-20T22:22:40.105 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: index_type: 0 2024-09-20T22:22:40.105 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: data_block_index_type: 0 2024-09-20T22:22:40.105 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: index_shortening: 1 2024-09-20T22:22:40.105 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-20T22:22:40.105 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: checksum: 4 2024-09-20T22:22:40.105 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: no_block_cache: 0 2024-09-20T22:22:40.105 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: block_cache: 0x55e96481b350 2024-09-20T22:22:40.105 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: block_cache_name: BinnedLRUCache 2024-09-20T22:22:40.106 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: block_cache_options: 2024-09-20T22:22:40.106 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: capacity : 536870912 2024-09-20T22:22:40.106 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: num_shard_bits : 4 2024-09-20T22:22:40.106 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: strict_capacity_limit : 0 2024-09-20T22:22:40.106 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: high_pri_pool_ratio: 0.000 2024-09-20T22:22:40.106 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: block_cache_compressed: (nil) 2024-09-20T22:22:40.106 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: persistent_cache: (nil) 2024-09-20T22:22:40.106 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: block_size: 4096 2024-09-20T22:22:40.106 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: block_size_deviation: 10 2024-09-20T22:22:40.106 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: block_restart_interval: 16 2024-09-20T22:22:40.106 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: index_block_restart_interval: 1 2024-09-20T22:22:40.106 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: metadata_block_size: 4096 2024-09-20T22:22:40.106 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: partition_filters: 0 2024-09-20T22:22:40.106 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: use_delta_encoding: 1 2024-09-20T22:22:40.107 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: filter_policy: bloomfilter 2024-09-20T22:22:40.107 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: whole_key_filtering: 1 2024-09-20T22:22:40.107 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: verify_compression: 0 2024-09-20T22:22:40.107 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: read_amp_bytes_per_bit: 0 2024-09-20T22:22:40.107 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: format_version: 5 2024-09-20T22:22:40.107 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: enable_index_compression: 1 2024-09-20T22:22:40.107 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: block_align: 0 2024-09-20T22:22:40.107 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: max_auto_readahead_size: 262144 2024-09-20T22:22:40.107 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: prepopulate_block_cache: 0 2024-09-20T22:22:40.107 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: initial_auto_readahead_size: 8192 2024-09-20T22:22:40.108 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: num_file_reads_for_auto_readahead: 2 2024-09-20T22:22:40.108 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.write_buffer_size: 33554432 2024-09-20T22:22:40.108 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.max_write_buffer_number: 2 2024-09-20T22:22:40.108 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.compression: NoCompression 2024-09-20T22:22:40.108 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.bottommost_compression: Disabled 2024-09-20T22:22:40.108 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.prefix_extractor: nullptr 2024-09-20T22:22:40.108 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-20T22:22:40.108 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.num_levels: 7 2024-09-20T22:22:40.108 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-20T22:22:40.108 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-20T22:22:40.108 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-20T22:22:40.108 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-20T22:22:40.108 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-20T22:22:40.108 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-20T22:22:40.108 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-20T22:22:40.108 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-20T22:22:40.109 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-20T22:22:40.109 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-20T22:22:40.109 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-20T22:22:40.109 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-20T22:22:40.109 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-20T22:22:40.109 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.compression_opts.level: 32767 2024-09-20T22:22:40.109 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.compression_opts.strategy: 0 2024-09-20T22:22:40.109 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-20T22:22:40.109 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-20T22:22:40.109 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-20T22:22:40.109 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-20T22:22:40.109 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.compression_opts.enabled: false 2024-09-20T22:22:40.109 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-20T22:22:40.109 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-20T22:22:40.109 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-20T22:22:40.110 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-20T22:22:40.110 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.target_file_size_base: 67108864 2024-09-20T22:22:40.110 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-20T22:22:40.110 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-20T22:22:40.110 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-20T22:22:40.110 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-20T22:22:40.110 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-20T22:22:40.110 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-20T22:22:40.110 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-20T22:22:40.110 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-20T22:22:40.110 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-20T22:22:40.110 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-20T22:22:40.110 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-20T22:22:40.111 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-20T22:22:40.111 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-20T22:22:40.111 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-20T22:22:40.111 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.arena_block_size: 1048576 2024-09-20T22:22:40.111 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-20T22:22:40.111 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-20T22:22:40.111 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.disable_auto_compactions: 0 2024-09-20T22:22:40.111 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-20T22:22:40.111 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-20T22:22:40.111 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-20T22:22:40.111 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-20T22:22:40.111 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-20T22:22:40.111 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-20T22:22:40.111 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-20T22:22:40.112 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-20T22:22:40.112 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-20T22:22:40.112 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-20T22:22:40.112 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-20T22:22:40.112 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.inplace_update_support: 0 2024-09-20T22:22:40.112 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-20T22:22:40.112 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-20T22:22:40.112 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-20T22:22:40.112 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-20T22:22:40.112 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.bloom_locality: 0 2024-09-20T22:22:40.112 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.max_successive_merges: 0 2024-09-20T22:22:40.112 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-20T22:22:40.112 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.paranoid_file_checks: 0 2024-09-20T22:22:40.113 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.force_consistency_checks: 1 2024-09-20T22:22:40.113 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.report_bg_io_stats: 0 2024-09-20T22:22:40.113 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.ttl: 2592000 2024-09-20T22:22:40.113 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-20T22:22:40.113 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-20T22:22:40.113 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-20T22:22:40.113 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.enable_blob_files: false 2024-09-20T22:22:40.113 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.min_blob_size: 0 2024-09-20T22:22:40.113 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.blob_file_size: 268435456 2024-09-20T22:22:40.113 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-20T22:22:40.113 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-20T22:22:40.113 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-20T22:22:40.113 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-20T22:22:40.113 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-20T22:22:40.113 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.blob_file_starting_level: 0 2024-09-20T22:22:40.113 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-20T22:22:40.114 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 24.sst 2024-09-20T22:22:40.114 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-20T22:22:40.114 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi194/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 26, last_sequence is 12304, log_number is 22,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-20T22:22:40.114 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 22 2024-09-20T22:22:40.114 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 2e2abf58-56e1-4aa9-9186-d0e6bab1078a 2024-09-20T22:22:40.114 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726870959842547, "job": 1, "event": "recovery_started", "wal_files": [22]} 2024-09-20T22:22:40.114 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #22 mode 2 2024-09-20T22:22:40.114 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726870959856674, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 27, "file_size": 3229615, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12309, "largest_seqno": 13776, "table_properties": {"data_size": 3222999, "index_size": 4099, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1477, "raw_key_size": 13428, "raw_average_key_size": 23, "raw_value_size": 3210296, "raw_average_value_size": 5534, "num_data_blocks": 188, "num_entries": 580, "num_filter_entries": 580, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726870959, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "2e2abf58-56e1-4aa9-9186-d0e6bab1078a", "db_session_id": "63W7YL6L2RQF3ARMDO8Z", "orig_file_number": 27, "seqno_to_time_mapping": "N/A"}} 2024-09-20T22:22:40.114 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726870959856843, "job": 1, "event": "recovery_finished"} 2024-09-20T22:22:40.114 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: [db/version_set.cc:5047] Creating manifest 29 2024-09-20T22:22:40.115 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-20T22:22:40.115 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-smithi194/store.db/000022.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-20T22:22:40.115 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55e964848e00 2024-09-20T22:22:40.115 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: DB pointer 0x55e964944000 2024-09-20T22:22:40.115 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-20T22:22:40.115 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-20T22:22:40.116 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: ** DB Stats ** 2024-09-20T22:22:40.116 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-20T22:22:40.116 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-20T22:22:40.116 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-20T22:22:40.116 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-20T22:22:40.116 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-20T22:22:40.116 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-20T22:22:40.116 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-20T22:22:40.116 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: 2024-09-20T22:22:40.116 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: ** Compaction Stats [default] ** 2024-09-20T22:22:40.116 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-20T22:22:40.116 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-20T22:22:40.116 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: L0 1/0 3.08 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 515.2 0.01 0.00 1 0.006 0 0 0.0 0.0 2024-09-20T22:22:40.117 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: L6 1/0 8.14 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-20T22:22:40.117 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: Sum 2/0 11.22 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 515.2 0.01 0.00 1 0.006 0 0 0.0 0.0 2024-09-20T22:22:40.117 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 515.2 0.01 0.00 1 0.006 0 0 0.0 0.0 2024-09-20T22:22:40.117 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: 2024-09-20T22:22:40.117 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: ** Compaction Stats [default] ** 2024-09-20T22:22:40.117 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-20T22:22:40.117 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-20T22:22:40.117 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 515.2 0.01 0.00 1 0.006 0 0 0.0 0.0 2024-09-20T22:22:40.117 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: 2024-09-20T22:22:40.117 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-20T22:22:40.117 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: 2024-09-20T22:22:40.117 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-20T22:22:40.117 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: Flush(GB): cumulative 0.003, interval 0.003 2024-09-20T22:22:40.117 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-20T22:22:40.118 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-20T22:22:40.118 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-20T22:22:40.118 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-20T22:22:40.118 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: Cumulative compaction: 0.00 GB write, 180.29 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-20T22:22:40.118 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: Interval compaction: 0.00 GB write, 180.29 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-20T22:22:40.118 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-20T22:22:40.118 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: Block cache BinnedLRUCache@0x55e96481b350#2 capacity: 512.00 MB usage: 6.11 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2024-09-20T22:22:40.118 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.53 KB,0.000292063%) IndexBlock(1,4.58 KB,0.000873208%) Misc(1,0.00 KB,0%) 2024-09-20T22:22:40.118 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: 2024-09-20T22:22:40.118 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-20T22:22:40.118 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: starting mon.smithi194 rank 1 at public addrs [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] at bind addrs [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi194 fsid f9a6357a-779c-11ef-baf5-efdc52797490 2024-09-20T22:22:40.118 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: mon.smithi194@-1(???) e2 preinit fsid f9a6357a-779c-11ef-baf5-efdc52797490 2024-09-20T22:22:40.118 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: mon.smithi194@-1(???).mds e7 new map 2024-09-20T22:22:40.119 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: mon.smithi194@-1(???).mds e7 print_map 2024-09-20T22:22:40.119 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: e7 2024-09-20T22:22:40.119 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-20T22:22:40.119 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-20T22:22:40.119 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-20T22:22:40.119 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: legacy client fscid: 1 2024-09-20T22:22:40.119 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: 2024-09-20T22:22:40.119 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: Filesystem 'foofs' (1) 2024-09-20T22:22:40.119 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: fs_name foofs 2024-09-20T22:22:40.119 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: epoch 5 2024-09-20T22:22:40.119 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-20T22:22:40.119 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: created 2024-09-20T22:17:05.179585+0000 2024-09-20T22:22:40.119 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: modified 2024-09-20T22:17:11.201678+0000 2024-09-20T22:22:40.120 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: tableserver 0 2024-09-20T22:22:40.120 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: root 0 2024-09-20T22:22:40.120 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: session_timeout 60 2024-09-20T22:22:40.120 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: session_autoclose 300 2024-09-20T22:22:40.120 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: max_file_size 1099511627776 2024-09-20T22:22:40.120 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: max_xattr_size 65536 2024-09-20T22:22:40.120 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: required_client_features {} 2024-09-20T22:22:40.120 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: last_failure 0 2024-09-20T22:22:40.120 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: last_failure_osd_epoch 0 2024-09-20T22:22:40.120 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-20T22:22:40.120 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: max_mds 1 2024-09-20T22:22:40.120 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: in 0 2024-09-20T22:22:40.120 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: up {0=14488} 2024-09-20T22:22:40.120 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: failed 2024-09-20T22:22:40.121 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: damaged 2024-09-20T22:22:40.121 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: stopped 2024-09-20T22:22:40.121 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: data_pools [3] 2024-09-20T22:22:40.121 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: metadata_pool 2 2024-09-20T22:22:40.121 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: inline_data disabled 2024-09-20T22:22:40.121 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: balancer 2024-09-20T22:22:40.121 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: bal_rank_mask -1 2024-09-20T22:22:40.121 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: standby_count_wanted 1 2024-09-20T22:22:40.121 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: qdb_cluster leader: 0 members: 2024-09-20T22:22:40.121 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: [mds.foofs.smithi037.sdtedi{0:14488} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.37:6834/2142770746,v1:172.21.15.37:6835/2142770746] compat {c=[1],r=[1],i=[7ff]}] 2024-09-20T22:22:40.121 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: 2024-09-20T22:22:40.121 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: 2024-09-20T22:22:40.121 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: Standby daemons: 2024-09-20T22:22:40.121 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: 2024-09-20T22:22:40.122 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: [mds.foofs.smithi194.lkroso{-1:24305} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.194:6832/1702667629,v1:172.21.15.194:6833/1702667629] compat {c=[1],r=[1],i=[7ff]}] 2024-09-20T22:22:40.122 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: mon.smithi194@-1(???).osd e73 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-20T22:22:40.122 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: mon.smithi194@-1(???).osd e73 crush map has features 432629239337189376, adjusting msgr requires 2024-09-20T22:22:40.122 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: mon.smithi194@-1(???).osd e73 crush map has features 432629239337189376, adjusting msgr requires 2024-09-20T22:22:40.122 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: mon.smithi194@-1(???).osd e73 crush map has features 432629239337189376, adjusting msgr requires 2024-09-20T22:22:40.122 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 ceph-mon[51725]: mon.smithi194@-1(???).paxosservice(auth 1..30) refresh upgraded, format 0 -> 3 2024-09-20T22:22:40.122 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 bash[51708]: 97854807ba91a92fcb11515626eb160ee952ac236147761f68cbd8977b8382cd 2024-09-20T22:22:40.122 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:39 smithi194 systemd[1]: Started Ceph mon.smithi194 for f9a6357a-779c-11ef-baf5-efdc52797490. 2024-09-20T22:22:40.838 INFO:teuthology.orchestra.run.smithi037.stdout: 5 132694 11.49 MB/sec execute 178 sec latency 46.438 ms 2024-09-20T22:22:41.838 INFO:teuthology.orchestra.run.smithi037.stdout: 5 133002 11.48 MB/sec execute 179 sec latency 40.109 ms 2024-09-20T22:22:42.222 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:42 smithi037 ceph-mon[65640]: overall HEALTH_OK 2024-09-20T22:22:42.222 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:42 smithi037 ceph-mon[65640]: mon.smithi037 calling monitor election 2024-09-20T22:22:42.222 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:42 smithi037 ceph-mon[65640]: mon.smithi194 calling monitor election 2024-09-20T22:22:42.222 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:42 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mon metadata", "id": "smithi037"}]: dispatch 2024-09-20T22:22:42.222 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:42 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mon metadata", "id": "smithi194"}]: dispatch 2024-09-20T22:22:42.222 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:42 smithi037 ceph-mon[65640]: mon.smithi037 is new leader, mons smithi037,smithi194 in quorum (ranks 0,1) 2024-09-20T22:22:42.222 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:42 smithi037 ceph-mon[65640]: monmap epoch 3 2024-09-20T22:22:42.222 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:42 smithi037 ceph-mon[65640]: fsid f9a6357a-779c-11ef-baf5-efdc52797490 2024-09-20T22:22:42.222 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:42 smithi037 ceph-mon[65640]: last_changed 2024-09-20T22:22:41.060678+0000 2024-09-20T22:22:42.222 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:42 smithi037 ceph-mon[65640]: created 2024-09-20T22:09:36.885927+0000 2024-09-20T22:22:42.222 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:42 smithi037 ceph-mon[65640]: min_mon_release 19 (squid) 2024-09-20T22:22:42.223 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:42 smithi037 ceph-mon[65640]: election_strategy: 1 2024-09-20T22:22:42.223 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:42 smithi037 ceph-mon[65640]: 0: [v2:172.21.15.37:3300/0,v1:172.21.15.37:6789/0] mon.smithi037 2024-09-20T22:22:42.223 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:42 smithi037 ceph-mon[65640]: 1: [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] mon.smithi194 2024-09-20T22:22:42.223 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:42 smithi037 ceph-mon[65640]: fsmap foofs:1 {0=foofs.smithi037.sdtedi=up:active} 1 up:standby 2024-09-20T22:22:42.223 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:42 smithi037 ceph-mon[65640]: osdmap e73: 8 total, 8 up, 8 in 2024-09-20T22:22:42.223 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:42 smithi037 ceph-mon[65640]: mgrmap e43: smithi037.lykblh(active, since 19s), standbys: smithi194.bkcglt 2024-09-20T22:22:42.223 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:42 smithi037 ceph-mon[65640]: overall HEALTH_OK 2024-09-20T22:22:42.223 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:42 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:42.223 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:42 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:42.223 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:42 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:22:42.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:42 smithi194 ceph-mon[51725]: overall HEALTH_OK 2024-09-20T22:22:42.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:42 smithi194 ceph-mon[51725]: mon.smithi037 calling monitor election 2024-09-20T22:22:42.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:42 smithi194 ceph-mon[51725]: mon.smithi194 calling monitor election 2024-09-20T22:22:42.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:42 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mon metadata", "id": "smithi037"}]: dispatch 2024-09-20T22:22:42.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:42 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mon metadata", "id": "smithi194"}]: dispatch 2024-09-20T22:22:42.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:42 smithi194 ceph-mon[51725]: mon.smithi037 is new leader, mons smithi037,smithi194 in quorum (ranks 0,1) 2024-09-20T22:22:42.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:42 smithi194 ceph-mon[51725]: monmap epoch 3 2024-09-20T22:22:42.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:42 smithi194 ceph-mon[51725]: fsid f9a6357a-779c-11ef-baf5-efdc52797490 2024-09-20T22:22:42.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:42 smithi194 ceph-mon[51725]: last_changed 2024-09-20T22:22:41.060678+0000 2024-09-20T22:22:42.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:42 smithi194 ceph-mon[51725]: created 2024-09-20T22:09:36.885927+0000 2024-09-20T22:22:42.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:42 smithi194 ceph-mon[51725]: min_mon_release 19 (squid) 2024-09-20T22:22:42.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:42 smithi194 ceph-mon[51725]: election_strategy: 1 2024-09-20T22:22:42.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:42 smithi194 ceph-mon[51725]: 0: [v2:172.21.15.37:3300/0,v1:172.21.15.37:6789/0] mon.smithi037 2024-09-20T22:22:42.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:42 smithi194 ceph-mon[51725]: 1: [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] mon.smithi194 2024-09-20T22:22:42.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:42 smithi194 ceph-mon[51725]: fsmap foofs:1 {0=foofs.smithi037.sdtedi=up:active} 1 up:standby 2024-09-20T22:22:42.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:42 smithi194 ceph-mon[51725]: osdmap e73: 8 total, 8 up, 8 in 2024-09-20T22:22:42.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:42 smithi194 ceph-mon[51725]: mgrmap e43: smithi037.lykblh(active, since 19s), standbys: smithi194.bkcglt 2024-09-20T22:22:42.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:42 smithi194 ceph-mon[51725]: overall HEALTH_OK 2024-09-20T22:22:42.346 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:42 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:42.346 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:42 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:42.346 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:42 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:22:42.838 INFO:teuthology.orchestra.run.smithi037.stdout: 5 133492 11.51 MB/sec execute 180 sec latency 51.978 ms 2024-09-20T22:22:43.237 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:43 smithi037 ceph-mon[65640]: pgmap v13: 97 pgs: 97 active+clean; 333 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.3 MiB/s wr, 431 op/s 2024-09-20T22:22:43.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:43 smithi194 ceph-mon[51725]: pgmap v13: 97 pgs: 97 active+clean; 333 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.3 MiB/s wr, 431 op/s 2024-09-20T22:22:43.838 INFO:teuthology.orchestra.run.smithi037.stdout: 5 134036 11.51 MB/sec execute 181 sec latency 63.252 ms 2024-09-20T22:22:44.838 INFO:teuthology.orchestra.run.smithi037.stdout: 5 134252 11.48 MB/sec execute 182 sec latency 48.228 ms 2024-09-20T22:22:45.338 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:45 smithi037 ceph-mon[65640]: pgmap v14: 97 pgs: 97 active+clean; 329 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.4 MiB/s wr, 654 op/s 2024-09-20T22:22:45.338 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:45 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:45.338 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:45 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:45.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:45 smithi194 ceph-mon[51725]: pgmap v14: 97 pgs: 97 active+clean; 329 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.4 MiB/s wr, 654 op/s 2024-09-20T22:22:45.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:45 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:45.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:45 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:45.838 INFO:teuthology.orchestra.run.smithi037.stdout: 5 134528 11.48 MB/sec execute 183 sec latency 41.306 ms 2024-09-20T22:22:46.776 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:46 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:46.776 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:46 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:46.777 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:46 smithi037 ceph-mon[65640]: pgmap v15: 97 pgs: 97 active+clean; 329 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.8 MiB/s wr, 472 op/s 2024-09-20T22:22:46.838 INFO:teuthology.orchestra.run.smithi037.stdout: 5 134678 11.44 MB/sec execute 184 sec latency 268.964 ms 2024-09-20T22:22:46.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:46 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:46.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:46 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:46.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:46 smithi194 ceph-mon[51725]: pgmap v15: 97 pgs: 97 active+clean; 329 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.8 MiB/s wr, 472 op/s 2024-09-20T22:22:47.838 INFO:teuthology.orchestra.run.smithi037.stdout: 5 134849 11.40 MB/sec execute 185 sec latency 133.350 ms 2024-09-20T22:22:48.839 INFO:teuthology.orchestra.run.smithi037.stdout: 5 135042 11.37 MB/sec execute 186 sec latency 179.180 ms 2024-09-20T22:22:48.912 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:48 smithi037 ceph-mon[65640]: pgmap v16: 97 pgs: 97 active+clean; 329 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.8 MiB/s wr, 472 op/s 2024-09-20T22:22:48.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:48 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:48.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:48 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:48.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:48 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:22:48.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:48 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:22:48.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:48 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:48.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:48 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:49.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:48 smithi194 ceph-mon[51725]: pgmap v16: 97 pgs: 97 active+clean; 329 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.8 MiB/s wr, 472 op/s 2024-09-20T22:22:49.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:48 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:49.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:48 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:49.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:48 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:22:49.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:48 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:22:49.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:48 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:49.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:48 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:49.839 INFO:teuthology.orchestra.run.smithi037.stdout: 5 135283 11.36 MB/sec execute 187 sec latency 91.867 ms 2024-09-20T22:22:49.952 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:49 smithi037 ceph-mon[65640]: Reconfiguring mon.smithi037 (monmap changed)... 2024-09-20T22:22:49.953 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:49 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-20T22:22:49.953 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:49 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-20T22:22:49.953 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:49 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:22:49.953 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:49 smithi037 ceph-mon[65640]: Reconfiguring daemon mon.smithi037 on smithi037 2024-09-20T22:22:50.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:49 smithi194 ceph-mon[51725]: Reconfiguring mon.smithi037 (monmap changed)... 2024-09-20T22:22:50.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:49 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-20T22:22:50.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:49 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-20T22:22:50.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:49 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:22:50.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:49 smithi194 ceph-mon[51725]: Reconfiguring daemon mon.smithi037 on smithi037 2024-09-20T22:22:50.839 INFO:teuthology.orchestra.run.smithi037.stdout: 5 135469 11.34 MB/sec execute 188 sec latency 158.798 ms 2024-09-20T22:22:50.978 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:50 smithi037 ceph-mon[65640]: pgmap v17: 97 pgs: 97 active+clean; 335 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 6.5 MiB/s wr, 627 op/s 2024-09-20T22:22:51.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:50 smithi194 ceph-mon[51725]: pgmap v17: 97 pgs: 97 active+clean; 335 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 6.5 MiB/s wr, 627 op/s 2024-09-20T22:22:51.839 INFO:teuthology.orchestra.run.smithi037.stdout: 5 135754 11.30 MB/sec execute 189 sec latency 64.798 ms 2024-09-20T22:22:52.744 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:52 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:52.745 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:52 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:52.745 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:52 smithi037 ceph-mon[65640]: Reconfiguring mgr.smithi037.lykblh (monmap changed)... 2024-09-20T22:22:52.745 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:52 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi037.lykblh", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-20T22:22:52.745 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:52 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-20T22:22:52.745 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:52 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:22:52.746 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:52 smithi037 ceph-mon[65640]: Reconfiguring daemon mgr.smithi037.lykblh on smithi037 2024-09-20T22:22:52.746 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:52 smithi037 ceph-mon[65640]: pgmap v18: 97 pgs: 97 active+clean; 335 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.8 MiB/s wr, 379 op/s 2024-09-20T22:22:52.746 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:52 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:22:52.839 INFO:teuthology.orchestra.run.smithi037.stdout: 5 136262 11.27 MB/sec execute 190 sec latency 41.996 ms 2024-09-20T22:22:53.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:52 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:53.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:52 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:53.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:52 smithi194 ceph-mon[51725]: Reconfiguring mgr.smithi037.lykblh (monmap changed)... 2024-09-20T22:22:53.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:52 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi037.lykblh", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-20T22:22:53.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:52 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-20T22:22:53.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:52 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:22:53.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:52 smithi194 ceph-mon[51725]: Reconfiguring daemon mgr.smithi037.lykblh on smithi037 2024-09-20T22:22:53.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:52 smithi194 ceph-mon[51725]: pgmap v18: 97 pgs: 97 active+clean; 335 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.8 MiB/s wr, 379 op/s 2024-09-20T22:22:53.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:52 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:22:53.839 INFO:teuthology.orchestra.run.smithi037.stdout: 5 136562 11.26 MB/sec execute 191 sec latency 94.756 ms 2024-09-20T22:22:54.839 INFO:teuthology.orchestra.run.smithi037.stdout: 5 137129 11.30 MB/sec execute 192 sec latency 51.275 ms 2024-09-20T22:22:55.021 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:54 smithi037 ceph-mon[65640]: pgmap v19: 97 pgs: 97 active+clean; 336 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.4 MiB/s wr, 554 op/s 2024-09-20T22:22:55.021 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:54 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:55.021 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:54 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:55.022 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:54 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi037", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-20T22:22:55.022 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:54 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:22:55.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:54 smithi194 ceph-mon[51725]: pgmap v19: 97 pgs: 97 active+clean; 336 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.4 MiB/s wr, 554 op/s 2024-09-20T22:22:55.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:54 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:55.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:54 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:55.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:54 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi037", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-20T22:22:55.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:54 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:22:55.839 INFO:teuthology.orchestra.run.smithi037.stdout: 5 137596 11.30 MB/sec execute 193 sec latency 60.536 ms 2024-09-20T22:22:56.060 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:55 smithi037 ceph-mon[65640]: Reconfiguring crash.smithi037 (monmap changed)... 2024-09-20T22:22:56.060 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:55 smithi037 ceph-mon[65640]: Reconfiguring daemon crash.smithi037 on smithi037 2024-09-20T22:22:56.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:55 smithi194 ceph-mon[51725]: Reconfiguring crash.smithi037 (monmap changed)... 2024-09-20T22:22:56.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:55 smithi194 ceph-mon[51725]: Reconfiguring daemon crash.smithi037 on smithi037 2024-09-20T22:22:56.839 INFO:teuthology.orchestra.run.smithi037.stdout: 5 137807 11.27 MB/sec execute 194 sec latency 54.545 ms 2024-09-20T22:22:57.078 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:56 smithi037 ceph-mon[65640]: pgmap v20: 97 pgs: 97 active+clean; 336 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.3 MiB/s wr, 330 op/s 2024-09-20T22:22:57.245 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:56 smithi194 ceph-mon[51725]: pgmap v20: 97 pgs: 97 active+clean; 336 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.3 MiB/s wr, 330 op/s 2024-09-20T22:22:57.839 INFO:teuthology.orchestra.run.smithi037.stdout: 5 138050 11.26 MB/sec execute 195 sec latency 97.483 ms 2024-09-20T22:22:58.149 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:57 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:58.150 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:57 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:58.150 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:57 smithi037 ceph-mon[65640]: Reconfiguring osd.0 (monmap changed)... 2024-09-20T22:22:58.150 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:57 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-20T22:22:58.150 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:57 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:22:58.150 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:57 smithi037 ceph-mon[65640]: Reconfiguring daemon osd.0 on smithi037 2024-09-20T22:22:58.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:57 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:58.346 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:57 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:22:58.346 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:57 smithi194 ceph-mon[51725]: Reconfiguring osd.0 (monmap changed)... 2024-09-20T22:22:58.346 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:57 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-20T22:22:58.346 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:57 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:22:58.346 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:57 smithi194 ceph-mon[51725]: Reconfiguring daemon osd.0 on smithi037 2024-09-20T22:22:58.839 INFO:teuthology.orchestra.run.smithi037.stdout: 5 138180 11.21 MB/sec execute 196 sec latency 159.917 ms 2024-09-20T22:22:59.211 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:22:58 smithi037 ceph-mon[65640]: pgmap v21: 97 pgs: 97 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.8 MiB/s wr, 431 op/s 2024-09-20T22:22:59.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:22:58 smithi194 ceph-mon[51725]: pgmap v21: 97 pgs: 97 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.8 MiB/s wr, 431 op/s 2024-09-20T22:22:59.839 INFO:teuthology.orchestra.run.smithi037.stdout: 5 138621 11.23 MB/sec execute 197 sec latency 83.661 ms 2024-09-20T22:23:00.222 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:00 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:00.223 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:00 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:00.223 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:00 smithi037 ceph-mon[65640]: Reconfiguring osd.1 (monmap changed)... 2024-09-20T22:23:00.223 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:00 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-20T22:23:00.223 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:00 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:23:00.223 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:00 smithi037 ceph-mon[65640]: Reconfiguring daemon osd.1 on smithi037 2024-09-20T22:23:00.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:00 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:00.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:00 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:00.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:00 smithi194 ceph-mon[51725]: Reconfiguring osd.1 (monmap changed)... 2024-09-20T22:23:00.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:00 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-20T22:23:00.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:00 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:23:00.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:00 smithi194 ceph-mon[51725]: Reconfiguring daemon osd.1 on smithi037 2024-09-20T22:23:00.839 INFO:teuthology.orchestra.run.smithi037.stdout: 5 139040 11.25 MB/sec execute 198 sec latency 36.943 ms 2024-09-20T22:23:01.247 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:01 smithi037 ceph-mon[65640]: pgmap v22: 97 pgs: 97 active+clean; 337 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 5.6 MiB/s wr, 550 op/s 2024-09-20T22:23:01.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:01 smithi194 ceph-mon[51725]: pgmap v22: 97 pgs: 97 active+clean; 337 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 5.6 MiB/s wr, 550 op/s 2024-09-20T22:23:01.839 INFO:teuthology.orchestra.run.smithi037.stdout: 5 139366 11.22 MB/sec execute 199 sec latency 56.265 ms 2024-09-20T22:23:02.789 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:02 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:02.789 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:02 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:02.789 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:02 smithi037 ceph-mon[65640]: Reconfiguring osd.2 (monmap changed)... 2024-09-20T22:23:02.789 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:02 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-20T22:23:02.789 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:02 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:23:02.789 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:02 smithi037 ceph-mon[65640]: Reconfiguring daemon osd.2 on smithi037 2024-09-20T22:23:02.789 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:02 smithi037 ceph-mon[65640]: pgmap v23: 97 pgs: 97 active+clean; 337 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.9 MiB/s wr, 395 op/s 2024-09-20T22:23:02.839 INFO:teuthology.orchestra.run.smithi037.stdout: 5 139798 11.18 MB/sec execute 200 sec latency 58.608 ms 2024-09-20T22:23:03.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:02 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:03.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:02 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:03.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:02 smithi194 ceph-mon[51725]: Reconfiguring osd.2 (monmap changed)... 2024-09-20T22:23:03.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:02 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-20T22:23:03.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:02 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:23:03.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:02 smithi194 ceph-mon[51725]: Reconfiguring daemon osd.2 on smithi037 2024-09-20T22:23:03.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:02 smithi194 ceph-mon[51725]: pgmap v23: 97 pgs: 97 active+clean; 337 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.9 MiB/s wr, 395 op/s 2024-09-20T22:23:03.839 INFO:teuthology.orchestra.run.smithi037.stdout: 5 140447 11.24 MB/sec execute 201 sec latency 28.109 ms 2024-09-20T22:23:04.839 INFO:teuthology.orchestra.run.smithi037.stdout: 5 140965 11.24 MB/sec execute 202 sec latency 39.900 ms 2024-09-20T22:23:05.081 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:04 smithi037 ceph-mon[65640]: pgmap v24: 97 pgs: 97 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.9 MiB/s wr, 601 op/s 2024-09-20T22:23:05.081 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:04 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:05.082 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:04 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:05.082 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:04 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-20T22:23:05.082 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:04 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:23:05.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:04 smithi194 ceph-mon[51725]: pgmap v24: 97 pgs: 97 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.9 MiB/s wr, 601 op/s 2024-09-20T22:23:05.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:04 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:05.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:04 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:05.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:04 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-20T22:23:05.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:04 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:23:05.839 INFO:teuthology.orchestra.run.smithi037.stdout: 5 141404 11.24 MB/sec execute 203 sec latency 47.758 ms 2024-09-20T22:23:06.091 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:05 smithi037 ceph-mon[65640]: Reconfiguring osd.3 (monmap changed)... 2024-09-20T22:23:06.092 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:05 smithi037 ceph-mon[65640]: Reconfiguring daemon osd.3 on smithi037 2024-09-20T22:23:06.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:05 smithi194 ceph-mon[51725]: Reconfiguring osd.3 (monmap changed)... 2024-09-20T22:23:06.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:05 smithi194 ceph-mon[51725]: Reconfiguring daemon osd.3 on smithi037 2024-09-20T22:23:06.840 INFO:teuthology.orchestra.run.smithi037.stdout: 5 141839 11.26 MB/sec execute 204 sec latency 102.576 ms 2024-09-20T22:23:07.107 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:06 smithi037 ceph-mon[65640]: pgmap v25: 97 pgs: 97 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.3 MiB/s wr, 426 op/s 2024-09-20T22:23:07.108 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:06 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:23:07.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:06 smithi194 ceph-mon[51725]: pgmap v25: 97 pgs: 97 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.3 MiB/s wr, 426 op/s 2024-09-20T22:23:07.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:06 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:23:07.736 INFO:teuthology.orchestra.run.smithi037.stdout:true 2024-09-20T22:23:07.840 INFO:teuthology.orchestra.run.smithi037.stdout: 5 142224 11.27 MB/sec execute 205 sec latency 60.636 ms 2024-09-20T22:23:08.261 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:08 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:08.262 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:08 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:08.262 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:08 smithi194 ceph-mon[51725]: Reconfiguring mds.foofs.smithi037.sdtedi (monmap changed)... 2024-09-20T22:23:08.262 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:08 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi037.sdtedi", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-20T22:23:08.262 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:08 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:23:08.262 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:08 smithi194 ceph-mon[51725]: Reconfiguring daemon mds.foofs.smithi037.sdtedi on smithi037 2024-09-20T22:23:08.262 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:08 smithi194 ceph-mon[51725]: from='client.44101 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:23:08.262 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:08 smithi194 ceph-mon[51725]: pgmap v26: 97 pgs: 97 active+clean; 344 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 4.8 MiB/s wr, 529 op/s 2024-09-20T22:23:08.392 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:08 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:08.392 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:08 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:08.393 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:08 smithi037 ceph-mon[65640]: Reconfiguring mds.foofs.smithi037.sdtedi (monmap changed)... 2024-09-20T22:23:08.393 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:08 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi037.sdtedi", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-20T22:23:08.393 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:08 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:23:08.393 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:08 smithi037 ceph-mon[65640]: Reconfiguring daemon mds.foofs.smithi037.sdtedi on smithi037 2024-09-20T22:23:08.393 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:08 smithi037 ceph-mon[65640]: from='client.44101 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:23:08.393 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:08 smithi037 ceph-mon[65640]: pgmap v26: 97 pgs: 97 active+clean; 344 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 4.8 MiB/s wr, 529 op/s 2024-09-20T22:23:08.832 INFO:teuthology.orchestra.run.smithi037.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T22:23:08.833 INFO:teuthology.orchestra.run.smithi037.stdout:alertmanager.smithi037 smithi037 *:9093,9094 running (3m) 37s ago 11m 27.9M - 0.25.0 c8568f914cd2 d1f99b3cd707 2024-09-20T22:23:08.833 INFO:teuthology.orchestra.run.smithi037.stdout:crash.smithi037 smithi037 running (11m) 37s ago 11m 6400k - 17.2.0 e1d6a67b021e 88301022bdc8 2024-09-20T22:23:08.833 INFO:teuthology.orchestra.run.smithi037.stdout:crash.smithi194 smithi194 running (11m) 24s ago 11m 6405k - 17.2.0 e1d6a67b021e b553601d7ca5 2024-09-20T22:23:08.833 INFO:teuthology.orchestra.run.smithi037.stdout:grafana.smithi037 smithi037 *:3000 running (2m) 37s ago 11m 51.7M - dad864ee21e9 2306f3c2da19 2024-09-20T22:23:08.833 INFO:teuthology.orchestra.run.smithi037.stdout:mds.foofs.smithi037.sdtedi smithi037 running (6m) 37s ago 5m 37.2M - 17.2.0 e1d6a67b021e 0a7898d25b8d 2024-09-20T22:23:08.833 INFO:teuthology.orchestra.run.smithi037.stdout:mds.foofs.smithi194.lkroso smithi194 running (5m) 24s ago 5m 15.4M - 17.2.0 e1d6a67b021e 17736de30008 2024-09-20T22:23:08.833 INFO:teuthology.orchestra.run.smithi037.stdout:mgr.smithi037.lykblh smithi037 *:8443,9283,8765 running (2m) 37s ago 13m 475M - 19.1.1-367-g9e2ac70b cc9251b744a3 181e46fd9ba6 2024-09-20T22:23:08.833 INFO:teuthology.orchestra.run.smithi037.stdout:mgr.smithi194.bkcglt smithi194 *:8443,9283,8765 running (73s) 24s ago 11m 438M - 19.1.1-367-g9e2ac70b cc9251b744a3 1fa610c97517 2024-09-20T22:23:08.833 INFO:teuthology.orchestra.run.smithi037.stdout:mon.smithi037 smithi037 running (54s) 37s ago 13m 30.9M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 c0229eceb527 2024-09-20T22:23:08.833 INFO:teuthology.orchestra.run.smithi037.stdout:mon.smithi194 smithi194 running (29s) 24s ago 11m 24.4M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 97854807ba91 2024-09-20T22:23:08.833 INFO:teuthology.orchestra.run.smithi037.stdout:nfs.foo.0.0.smithi037.spabdc smithi037 *:2049 running (5m) 37s ago 5m 265M - 4.0 e1d6a67b021e 858ff9d558a2 2024-09-20T22:23:08.833 INFO:teuthology.orchestra.run.smithi037.stdout:nfs.foo.1.0.smithi194.qzjyej smithi194 *:2049 running (5m) 24s ago 5m 89.8M - 4.0 e1d6a67b021e c07fe0eb7c75 2024-09-20T22:23:08.833 INFO:teuthology.orchestra.run.smithi037.stdout:node-exporter.smithi037 smithi037 *:9100 running (2m) 37s ago 11m 9915k - 1.5.0 0da6a335fe13 433e6aaf6137 2024-09-20T22:23:08.834 INFO:teuthology.orchestra.run.smithi037.stdout:node-exporter.smithi194 smithi194 *:9100 running (2m) 24s ago 11m 16.3M - 1.5.0 0da6a335fe13 1d30bb810ad1 2024-09-20T22:23:08.834 INFO:teuthology.orchestra.run.smithi037.stdout:osd.0 smithi037 running (10m) 37s ago 10m 463M 1937M 17.2.0 e1d6a67b021e e54e84fd5030 2024-09-20T22:23:08.834 INFO:teuthology.orchestra.run.smithi037.stdout:osd.1 smithi037 running (9m) 37s ago 9m 403M 1937M 17.2.0 e1d6a67b021e 09647c133032 2024-09-20T22:23:08.834 INFO:teuthology.orchestra.run.smithi037.stdout:osd.2 smithi037 running (9m) 37s ago 9m 290M 1937M 17.2.0 e1d6a67b021e 6c15b1175256 2024-09-20T22:23:08.834 INFO:teuthology.orchestra.run.smithi037.stdout:osd.3 smithi037 running (8m) 37s ago 8m 426M 1937M 17.2.0 e1d6a67b021e 3a22952c6475 2024-09-20T22:23:08.834 INFO:teuthology.orchestra.run.smithi037.stdout:osd.4 smithi194 running (8m) 24s ago 8m 478M 2704M 17.2.0 e1d6a67b021e fa107be8915c 2024-09-20T22:23:08.834 INFO:teuthology.orchestra.run.smithi037.stdout:osd.5 smithi194 running (8m) 24s ago 8m 522M 2704M 17.2.0 e1d6a67b021e 290b21cbd131 2024-09-20T22:23:08.834 INFO:teuthology.orchestra.run.smithi037.stdout:osd.6 smithi194 running (7m) 24s ago 7m 432M 2704M 17.2.0 e1d6a67b021e 27f0720639ed 2024-09-20T22:23:08.834 INFO:teuthology.orchestra.run.smithi037.stdout:osd.7 smithi194 running (7m) 24s ago 7m 338M 2704M 17.2.0 e1d6a67b021e 62d163f4fd92 2024-09-20T22:23:08.834 INFO:teuthology.orchestra.run.smithi037.stdout:prometheus.smithi037 smithi037 *:9095 running (80s) 37s ago 10m 47.2M - 2.43.0 a07b618ecd1d 05b826dde2ff 2024-09-20T22:23:08.839 INFO:teuthology.orchestra.run.smithi037.stdout: 5 142569 11.27 MB/sec execute 206 sec latency 41.325 ms 2024-09-20T22:23:09.407 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:09 smithi037 ceph-mon[65640]: from='client.44105 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:23:09.407 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:09 smithi037 ceph-mon[65640]: from='client.34158 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:23:09.416 INFO:teuthology.orchestra.run.smithi037.stdout:{ 2024-09-20T22:23:09.416 INFO:teuthology.orchestra.run.smithi037.stdout: "mon": { 2024-09-20T22:23:09.416 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 2 2024-09-20T22:23:09.416 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:23:09.417 INFO:teuthology.orchestra.run.smithi037.stdout: "mgr": { 2024-09-20T22:23:09.417 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 2 2024-09-20T22:23:09.417 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:23:09.417 INFO:teuthology.orchestra.run.smithi037.stdout: "osd": { 2024-09-20T22:23:09.417 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-20T22:23:09.417 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:23:09.417 INFO:teuthology.orchestra.run.smithi037.stdout: "mds": { 2024-09-20T22:23:09.417 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T22:23:09.417 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:23:09.417 INFO:teuthology.orchestra.run.smithi037.stdout: "overall": { 2024-09-20T22:23:09.417 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-09-20T22:23:09.417 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 4 2024-09-20T22:23:09.417 INFO:teuthology.orchestra.run.smithi037.stdout: } 2024-09-20T22:23:09.417 INFO:teuthology.orchestra.run.smithi037.stdout:} 2024-09-20T22:23:09.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:09 smithi194 ceph-mon[51725]: from='client.44105 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:23:09.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:09 smithi194 ceph-mon[51725]: from='client.34158 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:23:09.840 INFO:teuthology.orchestra.run.smithi037.stdout: 5 142906 11.25 MB/sec execute 207 sec latency 53.351 ms 2024-09-20T22:23:09.987 INFO:teuthology.orchestra.run.smithi037.stdout:{ 2024-09-20T22:23:09.987 INFO:teuthology.orchestra.run.smithi037.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5", 2024-09-20T22:23:09.987 INFO:teuthology.orchestra.run.smithi037.stdout: "in_progress": true, 2024-09-20T22:23:09.987 INFO:teuthology.orchestra.run.smithi037.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-20T22:23:09.987 INFO:teuthology.orchestra.run.smithi037.stdout: "services_complete": [ 2024-09-20T22:23:09.988 INFO:teuthology.orchestra.run.smithi037.stdout: "mon", 2024-09-20T22:23:09.988 INFO:teuthology.orchestra.run.smithi037.stdout: "mgr" 2024-09-20T22:23:09.988 INFO:teuthology.orchestra.run.smithi037.stdout: ], 2024-09-20T22:23:09.988 INFO:teuthology.orchestra.run.smithi037.stdout: "progress": "4/23 daemons upgraded", 2024-09-20T22:23:09.988 INFO:teuthology.orchestra.run.smithi037.stdout: "message": "Currently upgrading mon daemons", 2024-09-20T22:23:09.988 INFO:teuthology.orchestra.run.smithi037.stdout: "is_paused": false 2024-09-20T22:23:09.988 INFO:teuthology.orchestra.run.smithi037.stdout:} 2024-09-20T22:23:10.372 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:10 smithi194 ceph-mon[51725]: from='client.? 172.21.15.37:0/3714634066' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:23:10.372 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:10 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:10.372 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:10 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:10.373 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:10 smithi194 ceph-mon[51725]: Reconfiguring crash.smithi194 (monmap changed)... 2024-09-20T22:23:10.373 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:10 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi194", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-20T22:23:10.373 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:10 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:23:10.373 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:10 smithi194 ceph-mon[51725]: Reconfiguring daemon crash.smithi194 on smithi194 2024-09-20T22:23:10.373 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:10 smithi194 ceph-mon[51725]: pgmap v27: 97 pgs: 97 active+clean; 341 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.1 MiB/s wr, 619 op/s 2024-09-20T22:23:10.373 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:10 smithi194 ceph-mon[51725]: from='client.44113 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:23:10.424 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:10 smithi037 ceph-mon[65640]: from='client.? 172.21.15.37:0/3714634066' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:23:10.424 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:10 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:10.424 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:10 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:10.424 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:10 smithi037 ceph-mon[65640]: Reconfiguring crash.smithi194 (monmap changed)... 2024-09-20T22:23:10.424 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:10 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi194", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-20T22:23:10.424 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:10 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:23:10.424 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:10 smithi037 ceph-mon[65640]: Reconfiguring daemon crash.smithi194 on smithi194 2024-09-20T22:23:10.424 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:10 smithi037 ceph-mon[65640]: pgmap v27: 97 pgs: 97 active+clean; 341 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.1 MiB/s wr, 619 op/s 2024-09-20T22:23:10.424 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:10 smithi037 ceph-mon[65640]: from='client.44113 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:23:10.567 INFO:teuthology.orchestra.run.smithi037.stdout:HEALTH_OK 2024-09-20T22:23:10.840 INFO:teuthology.orchestra.run.smithi037.stdout: 5 143254 11.21 MB/sec execute 208 sec latency 56.885 ms 2024-09-20T22:23:11.431 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:11 smithi037 ceph-mon[65640]: from='client.? 172.21.15.37:0/4143307125' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T22:23:11.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:11 smithi194 ceph-mon[51725]: from='client.? 172.21.15.37:0/4143307125' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T22:23:11.840 INFO:teuthology.orchestra.run.smithi037.stdout: 5 143512 11.19 MB/sec execute 209 sec latency 90.267 ms 2024-09-20T22:23:12.447 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:12 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:12.448 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:12 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:12.448 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:12 smithi037 ceph-mon[65640]: Reconfiguring mgr.smithi194.bkcglt (monmap changed)... 2024-09-20T22:23:12.448 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:12 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi194.bkcglt", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-20T22:23:12.448 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:12 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-20T22:23:12.448 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:12 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:23:12.448 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:12 smithi037 ceph-mon[65640]: Reconfiguring daemon mgr.smithi194.bkcglt on smithi194 2024-09-20T22:23:12.448 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:12 smithi037 ceph-mon[65640]: pgmap v28: 97 pgs: 97 active+clean; 341 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.2 MiB/s wr, 499 op/s 2024-09-20T22:23:12.840 INFO:teuthology.orchestra.run.smithi037.stdout: 5 143791 11.18 MB/sec execute 210 sec latency 47.686 ms 2024-09-20T22:23:12.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:12 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:12.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:12 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:12.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:12 smithi194 ceph-mon[51725]: Reconfiguring mgr.smithi194.bkcglt (monmap changed)... 2024-09-20T22:23:12.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:12 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi194.bkcglt", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-20T22:23:12.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:12 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-20T22:23:12.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:12 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:23:12.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:12 smithi194 ceph-mon[51725]: Reconfiguring daemon mgr.smithi194.bkcglt on smithi194 2024-09-20T22:23:12.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:12 smithi194 ceph-mon[51725]: pgmap v28: 97 pgs: 97 active+clean; 341 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.2 MiB/s wr, 499 op/s 2024-09-20T22:23:13.840 INFO:teuthology.orchestra.run.smithi037.stdout: 5 144081 11.17 MB/sec execute 211 sec latency 59.382 ms 2024-09-20T22:23:13.984 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:13 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:13.984 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:13 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:13.984 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:13 smithi037 ceph-mon[65640]: Reconfiguring mon.smithi194 (monmap changed)... 2024-09-20T22:23:13.984 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:13 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-20T22:23:13.984 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:13 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-20T22:23:13.984 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:13 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:23:13.984 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:13 smithi037 ceph-mon[65640]: Reconfiguring daemon mon.smithi194 on smithi194 2024-09-20T22:23:14.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:13 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:14.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:13 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:14.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:13 smithi194 ceph-mon[51725]: Reconfiguring mon.smithi194 (monmap changed)... 2024-09-20T22:23:14.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:13 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-20T22:23:14.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:13 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-20T22:23:14.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:13 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:23:14.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:13 smithi194 ceph-mon[51725]: Reconfiguring daemon mon.smithi194 on smithi194 2024-09-20T22:23:14.840 INFO:teuthology.orchestra.run.smithi037.stdout: 5 144464 11.16 MB/sec execute 212 sec latency 56.751 ms 2024-09-20T22:23:15.003 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:14 smithi037 ceph-mon[65640]: pgmap v29: 97 pgs: 97 active+clean; 333 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.2 MiB/s wr, 706 op/s 2024-09-20T22:23:15.003 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:14 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:15.003 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:14 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:15.003 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:14 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-20T22:23:15.004 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:14 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:23:15.165 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:14 smithi194 ceph-mon[51725]: pgmap v29: 97 pgs: 97 active+clean; 333 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.2 MiB/s wr, 706 op/s 2024-09-20T22:23:15.166 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:14 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:15.166 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:14 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:15.166 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:14 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-20T22:23:15.166 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:14 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:23:15.840 INFO:teuthology.orchestra.run.smithi037.stdout: 5 144882 11.16 MB/sec execute 213 sec latency 38.099 ms 2024-09-20T22:23:16.024 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:15 smithi037 ceph-mon[65640]: Reconfiguring osd.4 (monmap changed)... 2024-09-20T22:23:16.025 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:15 smithi037 ceph-mon[65640]: Reconfiguring daemon osd.4 on smithi194 2024-09-20T22:23:16.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:15 smithi194 ceph-mon[51725]: Reconfiguring osd.4 (monmap changed)... 2024-09-20T22:23:16.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:15 smithi194 ceph-mon[51725]: Reconfiguring daemon osd.4 on smithi194 2024-09-20T22:23:16.840 INFO:teuthology.orchestra.run.smithi037.stdout: 5 145391 11.18 MB/sec execute 214 sec latency 33.888 ms 2024-09-20T22:23:17.082 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:17 smithi037 ceph-mon[65640]: pgmap v30: 97 pgs: 97 active+clean; 333 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.2 MiB/s wr, 500 op/s 2024-09-20T22:23:17.082 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:17 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:17.082 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:17 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:17.082 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:17 smithi037 ceph-mon[65640]: Reconfiguring osd.5 (monmap changed)... 2024-09-20T22:23:17.082 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:17 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-20T22:23:17.082 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:17 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:23:17.082 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:17 smithi037 ceph-mon[65640]: Reconfiguring daemon osd.5 on smithi194 2024-09-20T22:23:17.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:17 smithi194 ceph-mon[51725]: pgmap v30: 97 pgs: 97 active+clean; 333 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.2 MiB/s wr, 500 op/s 2024-09-20T22:23:17.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:17 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:17.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:17 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:17.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:17 smithi194 ceph-mon[51725]: Reconfiguring osd.5 (monmap changed)... 2024-09-20T22:23:17.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:17 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-20T22:23:17.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:17 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:23:17.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:17 smithi194 ceph-mon[51725]: Reconfiguring daemon osd.5 on smithi194 2024-09-20T22:23:17.840 INFO:teuthology.orchestra.run.smithi037.stdout: 5 145749 11.18 MB/sec execute 215 sec latency 120.015 ms 2024-09-20T22:23:18.620 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:18 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:18.620 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:18 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:18.621 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:18 smithi037 ceph-mon[65640]: Reconfiguring osd.6 (monmap changed)... 2024-09-20T22:23:18.621 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:18 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-20T22:23:18.621 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:18 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:23:18.621 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:18 smithi037 ceph-mon[65640]: Reconfiguring daemon osd.6 on smithi194 2024-09-20T22:23:18.621 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:18 smithi037 ceph-mon[65640]: pgmap v31: 97 pgs: 97 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 5.6 MiB/s wr, 572 op/s 2024-09-20T22:23:18.840 INFO:teuthology.orchestra.run.smithi037.stdout: 5 146131 11.19 MB/sec execute 216 sec latency 55.948 ms 2024-09-20T22:23:18.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:18 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:18.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:18 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:18.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:18 smithi194 ceph-mon[51725]: Reconfiguring osd.6 (monmap changed)... 2024-09-20T22:23:18.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:18 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-20T22:23:18.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:18 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:23:18.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:18 smithi194 ceph-mon[51725]: Reconfiguring daemon osd.6 on smithi194 2024-09-20T22:23:18.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:18 smithi194 ceph-mon[51725]: pgmap v31: 97 pgs: 97 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 5.6 MiB/s wr, 572 op/s 2024-09-20T22:23:19.840 INFO:teuthology.orchestra.run.smithi037.stdout: 5 146502 11.17 MB/sec execute 217 sec latency 41.120 ms 2024-09-20T22:23:20.142 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:20 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:20.142 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:20 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:20.142 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:20 smithi037 ceph-mon[65640]: Reconfiguring osd.7 (monmap changed)... 2024-09-20T22:23:20.143 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:20 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-20T22:23:20.143 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:20 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:23:20.143 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:20 smithi037 ceph-mon[65640]: Reconfiguring daemon osd.7 on smithi194 2024-09-20T22:23:20.521 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:20 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:20.522 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:20 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:20.522 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:20 smithi194 ceph-mon[51725]: Reconfiguring osd.7 (monmap changed)... 2024-09-20T22:23:20.522 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:20 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-20T22:23:20.522 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:20 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:23:20.522 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:20 smithi194 ceph-mon[51725]: Reconfiguring daemon osd.7 on smithi194 2024-09-20T22:23:20.840 INFO:teuthology.orchestra.run.smithi037.stdout: 5 146831 11.13 MB/sec execute 218 sec latency 63.413 ms 2024-09-20T22:23:21.162 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:21 smithi037 ceph-mon[65640]: pgmap v32: 97 pgs: 97 active+clean; 345 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 7.2 MiB/s wr, 617 op/s 2024-09-20T22:23:21.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:21 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:21.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:21 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:21.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:21 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi194.lkroso", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-20T22:23:21.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:21 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:23:21.317 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:21 smithi194 ceph-mon[51725]: pgmap v32: 97 pgs: 97 active+clean; 345 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 7.2 MiB/s wr, 617 op/s 2024-09-20T22:23:21.317 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:21 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:21.317 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:21 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:21.317 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:21 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi194.lkroso", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-20T22:23:21.317 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:21 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:23:21.840 INFO:teuthology.orchestra.run.smithi037.stdout: 5 147037 11.10 MB/sec execute 219 sec latency 69.039 ms 2024-09-20T22:23:22.196 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:22 smithi037 ceph-mon[65640]: Reconfiguring mds.foofs.smithi194.lkroso (monmap changed)... 2024-09-20T22:23:22.197 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:22 smithi037 ceph-mon[65640]: Reconfiguring daemon mds.foofs.smithi194.lkroso on smithi194 2024-09-20T22:23:22.197 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:22 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:23:22.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:22 smithi194 ceph-mon[51725]: Reconfiguring mds.foofs.smithi194.lkroso (monmap changed)... 2024-09-20T22:23:22.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:22 smithi194 ceph-mon[51725]: Reconfiguring daemon mds.foofs.smithi194.lkroso on smithi194 2024-09-20T22:23:22.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:22 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:23:22.840 INFO:teuthology.orchestra.run.smithi037.stdout: 5 147253 11.09 MB/sec execute 220 sec latency 80.022 ms 2024-09-20T22:23:23.237 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:23 smithi037 ceph-mon[65640]: pgmap v33: 97 pgs: 97 active+clean; 345 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.4 MiB/s wr, 427 op/s 2024-09-20T22:23:23.238 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:23 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:23.238 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:23 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:23.238 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:23 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:23:23.238 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:23 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:23:23.238 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:23 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:23:23.238 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:23 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:23.238 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:23 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi037"}]: dispatch 2024-09-20T22:23:23.238 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:23 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi037"}]': finished 2024-09-20T22:23:23.238 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:23 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi194"}]: dispatch 2024-09-20T22:23:23.238 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:23 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi194"}]': finished 2024-09-20T22:23:23.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:23 smithi194 ceph-mon[51725]: pgmap v33: 97 pgs: 97 active+clean; 345 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.4 MiB/s wr, 427 op/s 2024-09-20T22:23:23.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:23 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:23.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:23 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:23.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:23 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:23:23.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:23 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:23:23.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:23 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:23:23.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:23 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:23.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:23 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi037"}]: dispatch 2024-09-20T22:23:23.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:23 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi037"}]': finished 2024-09-20T22:23:23.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:23 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi194"}]: dispatch 2024-09-20T22:23:23.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:23 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi194"}]': finished 2024-09-20T22:23:23.840 INFO:teuthology.orchestra.run.smithi037.stdout: 5 147678 11.11 MB/sec execute 221 sec latency 46.029 ms 2024-09-20T22:23:24.255 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:24 smithi037 ceph-mon[65640]: Upgrade: Setting container_image for all mon 2024-09-20T22:23:24.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:24 smithi194 ceph-mon[51725]: Upgrade: Setting container_image for all mon 2024-09-20T22:23:24.840 INFO:teuthology.orchestra.run.smithi037.stdout: 5 148225 11.12 MB/sec execute 222 sec latency 27.032 ms 2024-09-20T22:23:25.263 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:25 smithi037 ceph-mon[65640]: pgmap v34: 97 pgs: 97 active+clean; 330 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.3 MiB/s wr, 635 op/s 2024-09-20T22:23:25.264 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:25 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:25.264 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:25 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi037", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-20T22:23:25.264 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:25 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:23:25.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:25 smithi194 ceph-mon[51725]: pgmap v34: 97 pgs: 97 active+clean; 330 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.3 MiB/s wr, 635 op/s 2024-09-20T22:23:25.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:25 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:25.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:25 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi037", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-20T22:23:25.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:25 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:23:25.840 INFO:teuthology.orchestra.run.smithi037.stdout: 5 148492 11.09 MB/sec execute 223 sec latency 53.285 ms 2024-09-20T22:23:26.281 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:26 smithi037 ceph-mon[65640]: Upgrade: Updating crash.smithi037 (1/2) 2024-09-20T22:23:26.281 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:26 smithi037 ceph-mon[65640]: Deploying daemon crash.smithi037 on smithi037 2024-09-20T22:23:26.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:26 smithi194 ceph-mon[51725]: Upgrade: Updating crash.smithi037 (1/2) 2024-09-20T22:23:26.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:26 smithi194 ceph-mon[51725]: Deploying daemon crash.smithi037 on smithi037 2024-09-20T22:23:26.840 INFO:teuthology.orchestra.run.smithi037.stdout: 5 148814 11.09 MB/sec execute 224 sec latency 52.270 ms 2024-09-20T22:23:27.308 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:27 smithi037 ceph-mon[65640]: pgmap v35: 97 pgs: 97 active+clean; 330 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.3 MiB/s wr, 427 op/s 2024-09-20T22:23:27.495 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:27 smithi194 ceph-mon[51725]: pgmap v35: 97 pgs: 97 active+clean; 330 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.3 MiB/s wr, 427 op/s 2024-09-20T22:23:27.841 INFO:teuthology.orchestra.run.smithi037.stdout: 5 149332 11.12 MB/sec execute 225 sec latency 34.486 ms 2024-09-20T22:23:28.841 INFO:teuthology.orchestra.run.smithi037.stdout: 5 149907 11.14 MB/sec execute 226 sec latency 90.932 ms 2024-09-20T22:23:29.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:29 smithi194 ceph-mon[51725]: pgmap v36: 97 pgs: 97 active+clean; 332 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 5.9 MiB/s wr, 618 op/s 2024-09-20T22:23:29.368 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:29 smithi037 ceph-mon[65640]: pgmap v36: 97 pgs: 97 active+clean; 332 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 5.9 MiB/s wr, 618 op/s 2024-09-20T22:23:29.840 INFO:teuthology.orchestra.run.smithi037.stdout: 5 150471 11.12 MB/sec execute 227 sec latency 59.217 ms 2024-09-20T22:23:30.840 INFO:teuthology.orchestra.run.smithi037.stdout: 5 150792 11.12 MB/sec execute 228 sec latency 52.691 ms 2024-09-20T22:23:31.158 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:31 smithi037 ceph-mon[65640]: pgmap v37: 97 pgs: 97 active+clean; 337 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 6.4 MiB/s wr, 583 op/s 2024-09-20T22:23:31.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:31 smithi194 ceph-mon[51725]: pgmap v37: 97 pgs: 97 active+clean; 337 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 6.4 MiB/s wr, 583 op/s 2024-09-20T22:23:31.840 INFO:teuthology.orchestra.run.smithi037.stdout: 5 151325 11.14 MB/sec execute 229 sec latency 43.282 ms 2024-09-20T22:23:32.841 INFO:teuthology.orchestra.run.smithi037.stdout: 5 151865 11.15 MB/sec execute 230 sec latency 33.171 ms 2024-09-20T22:23:33.172 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:32 smithi037 ceph-mon[65640]: pgmap v38: 97 pgs: 97 active+clean; 337 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.4 MiB/s wr, 436 op/s 2024-09-20T22:23:33.172 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:32 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:33.173 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:32 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:33.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:32 smithi194 ceph-mon[51725]: pgmap v38: 97 pgs: 97 active+clean; 337 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.4 MiB/s wr, 436 op/s 2024-09-20T22:23:33.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:32 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:33.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:32 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:33.840 INFO:teuthology.orchestra.run.smithi037.stdout: 5 152226 11.15 MB/sec execute 231 sec latency 34.551 ms 2024-09-20T22:23:34.695 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:34 smithi037 ceph-mon[65640]: Upgrade: Updating crash.smithi194 (2/2) 2024-09-20T22:23:34.696 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:34 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:34.696 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:34 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi194", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-20T22:23:34.696 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:34 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:23:34.696 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:34 smithi037 ceph-mon[65640]: Deploying daemon crash.smithi194 on smithi194 2024-09-20T22:23:34.696 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:34 smithi037 ceph-mon[65640]: pgmap v39: 97 pgs: 97 active+clean; 330 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.2 MiB/s wr, 740 op/s 2024-09-20T22:23:34.841 INFO:teuthology.orchestra.run.smithi037.stdout: 5 152814 11.18 MB/sec execute 232 sec latency 31.222 ms 2024-09-20T22:23:34.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:34 smithi194 ceph-mon[51725]: Upgrade: Updating crash.smithi194 (2/2) 2024-09-20T22:23:34.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:34 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:34.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:34 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi194", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-20T22:23:34.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:34 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:23:34.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:34 smithi194 ceph-mon[51725]: Deploying daemon crash.smithi194 on smithi194 2024-09-20T22:23:34.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:34 smithi194 ceph-mon[51725]: pgmap v39: 97 pgs: 97 active+clean; 330 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.2 MiB/s wr, 740 op/s 2024-09-20T22:23:35.841 INFO:teuthology.orchestra.run.smithi037.stdout: 5 153393 11.22 MB/sec execute 233 sec latency 43.469 ms 2024-09-20T22:23:36.841 INFO:teuthology.orchestra.run.smithi037.stdout: 5 153884 11.19 MB/sec execute 234 sec latency 50.764 ms 2024-09-20T22:23:36.961 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:36 smithi037 ceph-mon[65640]: pgmap v40: 97 pgs: 97 active+clean; 330 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.3 MiB/s wr, 533 op/s 2024-09-20T22:23:36.962 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:36 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:23:36.986 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:36 smithi194 ceph-mon[51725]: pgmap v40: 97 pgs: 97 active+clean; 330 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.3 MiB/s wr, 533 op/s 2024-09-20T22:23:36.986 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:36 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:23:37.841 INFO:teuthology.orchestra.run.smithi037.stdout: 5 154167 11.17 MB/sec execute 235 sec latency 37.052 ms 2024-09-20T22:23:38.841 INFO:teuthology.orchestra.run.smithi037.stdout: 5 154544 11.19 MB/sec execute 236 sec latency 60.430 ms 2024-09-20T22:23:39.263 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:39 smithi037 ceph-mon[65640]: pgmap v41: 97 pgs: 97 active+clean; 332 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.3 MiB/s wr, 779 op/s 2024-09-20T22:23:39.264 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:39 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:39.264 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:39 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:39.264 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:39 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:23:39.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:39 smithi194 ceph-mon[51725]: pgmap v41: 97 pgs: 97 active+clean; 332 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.3 MiB/s wr, 779 op/s 2024-09-20T22:23:39.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:39 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:39.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:39 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:39.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:39 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:23:39.841 INFO:teuthology.orchestra.run.smithi037.stdout: 5 155252 11.21 MB/sec execute 237 sec latency 43.433 ms 2024-09-20T22:23:40.841 INFO:teuthology.orchestra.run.smithi037.stdout: 5 155639 11.20 MB/sec execute 238 sec latency 24.167 ms 2024-09-20T22:23:41.220 INFO:teuthology.orchestra.run.smithi037.stdout:true 2024-09-20T22:23:41.276 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:41 smithi037 ceph-mon[65640]: pgmap v42: 97 pgs: 97 active+clean; 334 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 6.9 MiB/s wr, 637 op/s 2024-09-20T22:23:41.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:41 smithi194 ceph-mon[51725]: pgmap v42: 97 pgs: 97 active+clean; 334 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 6.9 MiB/s wr, 637 op/s 2024-09-20T22:23:41.841 INFO:teuthology.orchestra.run.smithi037.stdout: 5 156173 11.23 MB/sec execute 239 sec latency 46.147 ms 2024-09-20T22:23:42.397 INFO:teuthology.orchestra.run.smithi037.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T22:23:42.397 INFO:teuthology.orchestra.run.smithi037.stdout:alertmanager.smithi037 smithi037 *:9093,9094 running (3m) 71s ago 12m 27.9M - 0.25.0 c8568f914cd2 d1f99b3cd707 2024-09-20T22:23:42.398 INFO:teuthology.orchestra.run.smithi037.stdout:crash.smithi037 smithi037 starting - - - - 2024-09-20T22:23:42.398 INFO:teuthology.orchestra.run.smithi037.stdout:crash.smithi194 smithi194 running (4s) 0s ago 11m 6895k - 19.1.1-367-g9e2ac70b cc9251b744a3 4d169d1309bc 2024-09-20T22:23:42.398 INFO:teuthology.orchestra.run.smithi037.stdout:grafana.smithi037 smithi037 *:3000 running (3m) 71s ago 12m 51.7M - dad864ee21e9 2306f3c2da19 2024-09-20T22:23:42.398 INFO:teuthology.orchestra.run.smithi037.stdout:mds.foofs.smithi037.sdtedi smithi037 running (6m) 71s ago 6m 37.2M - 17.2.0 e1d6a67b021e 0a7898d25b8d 2024-09-20T22:23:42.398 INFO:teuthology.orchestra.run.smithi037.stdout:mds.foofs.smithi194.lkroso smithi194 running (6m) 0s ago 6m 15.6M - 17.2.0 e1d6a67b021e 17736de30008 2024-09-20T22:23:42.398 INFO:teuthology.orchestra.run.smithi037.stdout:mgr.smithi037.lykblh smithi037 *:8443,9283,8765 running (2m) 71s ago 13m 475M - 19.1.1-367-g9e2ac70b cc9251b744a3 181e46fd9ba6 2024-09-20T22:23:42.398 INFO:teuthology.orchestra.run.smithi037.stdout:mgr.smithi194.bkcglt smithi194 *:8443,9283,8765 running (107s) 0s ago 11m 439M - 19.1.1-367-g9e2ac70b cc9251b744a3 1fa610c97517 2024-09-20T22:23:42.398 INFO:teuthology.orchestra.run.smithi037.stdout:mon.smithi037 smithi037 running (88s) 71s ago 13m 30.9M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 c0229eceb527 2024-09-20T22:23:42.398 INFO:teuthology.orchestra.run.smithi037.stdout:mon.smithi194 smithi194 running (63s) 0s ago 11m 37.2M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 97854807ba91 2024-09-20T22:23:42.398 INFO:teuthology.orchestra.run.smithi037.stdout:nfs.foo.0.0.smithi037.spabdc smithi037 *:2049 running (6m) 71s ago 6m 265M - 4.0 e1d6a67b021e 858ff9d558a2 2024-09-20T22:23:42.398 INFO:teuthology.orchestra.run.smithi037.stdout:nfs.foo.1.0.smithi194.qzjyej smithi194 *:2049 running (6m) 0s ago 6m 89.8M - 4.0 e1d6a67b021e c07fe0eb7c75 2024-09-20T22:23:42.398 INFO:teuthology.orchestra.run.smithi037.stdout:node-exporter.smithi037 smithi037 *:9100 running (3m) 71s ago 12m 9915k - 1.5.0 0da6a335fe13 433e6aaf6137 2024-09-20T22:23:42.398 INFO:teuthology.orchestra.run.smithi037.stdout:node-exporter.smithi194 smithi194 *:9100 running (2m) 0s ago 11m 15.7M - 1.5.0 0da6a335fe13 1d30bb810ad1 2024-09-20T22:23:42.398 INFO:teuthology.orchestra.run.smithi037.stdout:osd.0 smithi037 running (10m) 71s ago 10m 463M 1937M 17.2.0 e1d6a67b021e e54e84fd5030 2024-09-20T22:23:42.399 INFO:teuthology.orchestra.run.smithi037.stdout:osd.1 smithi037 running (10m) 71s ago 10m 403M 1937M 17.2.0 e1d6a67b021e 09647c133032 2024-09-20T22:23:42.399 INFO:teuthology.orchestra.run.smithi037.stdout:osd.2 smithi037 running (9m) 71s ago 9m 290M 1937M 17.2.0 e1d6a67b021e 6c15b1175256 2024-09-20T22:23:42.399 INFO:teuthology.orchestra.run.smithi037.stdout:osd.3 smithi037 running (9m) 71s ago 9m 426M 1937M 17.2.0 e1d6a67b021e 3a22952c6475 2024-09-20T22:23:42.399 INFO:teuthology.orchestra.run.smithi037.stdout:osd.4 smithi194 running (9m) 0s ago 9m 513M 2704M 17.2.0 e1d6a67b021e fa107be8915c 2024-09-20T22:23:42.399 INFO:teuthology.orchestra.run.smithi037.stdout:osd.5 smithi194 running (8m) 0s ago 8m 324M 2704M 17.2.0 e1d6a67b021e 290b21cbd131 2024-09-20T22:23:42.399 INFO:teuthology.orchestra.run.smithi037.stdout:osd.6 smithi194 running (8m) 0s ago 8m 511M 2704M 17.2.0 e1d6a67b021e 27f0720639ed 2024-09-20T22:23:42.399 INFO:teuthology.orchestra.run.smithi037.stdout:osd.7 smithi194 running (8m) 0s ago 8m 406M 2704M 17.2.0 e1d6a67b021e 62d163f4fd92 2024-09-20T22:23:42.399 INFO:teuthology.orchestra.run.smithi037.stdout:prometheus.smithi037 smithi037 *:9095 running (113s) 71s ago 11m 47.2M - 2.43.0 a07b618ecd1d 05b826dde2ff 2024-09-20T22:23:42.718 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:42 smithi037 ceph-mon[65640]: from='client.44121 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:23:42.719 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:42 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:42.719 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:42 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:42.719 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:42 smithi037 ceph-mon[65640]: from='client.34176 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:23:42.719 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:42 smithi037 ceph-mon[65640]: pgmap v43: 97 pgs: 97 active+clean; 334 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 5.9 MiB/s wr, 599 op/s 2024-09-20T22:23:42.719 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:42 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:42.719 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:42 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:42.841 INFO:teuthology.orchestra.run.smithi037.stdout: 5 156587 11.25 MB/sec execute 240 sec latency 45.989 ms 2024-09-20T22:23:43.048 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:42 smithi194 ceph-mon[51725]: from='client.44121 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:23:43.048 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:42 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:43.048 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:42 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:43.048 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:42 smithi194 ceph-mon[51725]: from='client.34176 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:23:43.048 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:42 smithi194 ceph-mon[51725]: pgmap v43: 97 pgs: 97 active+clean; 334 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 5.9 MiB/s wr, 599 op/s 2024-09-20T22:23:43.048 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:42 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:43.048 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:42 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:43.154 INFO:teuthology.orchestra.run.smithi037.stdout:{ 2024-09-20T22:23:43.154 INFO:teuthology.orchestra.run.smithi037.stdout: "mon": { 2024-09-20T22:23:43.154 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 2 2024-09-20T22:23:43.154 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:23:43.155 INFO:teuthology.orchestra.run.smithi037.stdout: "mgr": { 2024-09-20T22:23:43.155 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 2 2024-09-20T22:23:43.155 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:23:43.155 INFO:teuthology.orchestra.run.smithi037.stdout: "osd": { 2024-09-20T22:23:43.155 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-20T22:23:43.155 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:23:43.155 INFO:teuthology.orchestra.run.smithi037.stdout: "mds": { 2024-09-20T22:23:43.155 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T22:23:43.155 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:23:43.155 INFO:teuthology.orchestra.run.smithi037.stdout: "overall": { 2024-09-20T22:23:43.155 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-09-20T22:23:43.155 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 4 2024-09-20T22:23:43.156 INFO:teuthology.orchestra.run.smithi037.stdout: } 2024-09-20T22:23:43.156 INFO:teuthology.orchestra.run.smithi037.stdout:} 2024-09-20T22:23:43.735 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:43 smithi037 ceph-mon[65640]: from='client.34180 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:23:43.736 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:43 smithi037 ceph-mon[65640]: from='client.? 172.21.15.37:0/1358906011' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:23:43.750 INFO:teuthology.orchestra.run.smithi037.stdout:{ 2024-09-20T22:23:43.751 INFO:teuthology.orchestra.run.smithi037.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5", 2024-09-20T22:23:43.751 INFO:teuthology.orchestra.run.smithi037.stdout: "in_progress": true, 2024-09-20T22:23:43.751 INFO:teuthology.orchestra.run.smithi037.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-20T22:23:43.751 INFO:teuthology.orchestra.run.smithi037.stdout: "services_complete": [ 2024-09-20T22:23:43.751 INFO:teuthology.orchestra.run.smithi037.stdout: "mon", 2024-09-20T22:23:43.751 INFO:teuthology.orchestra.run.smithi037.stdout: "mgr" 2024-09-20T22:23:43.751 INFO:teuthology.orchestra.run.smithi037.stdout: ], 2024-09-20T22:23:43.751 INFO:teuthology.orchestra.run.smithi037.stdout: "progress": "5/23 daemons upgraded", 2024-09-20T22:23:43.751 INFO:teuthology.orchestra.run.smithi037.stdout: "message": "Currently upgrading crash daemons", 2024-09-20T22:23:43.751 INFO:teuthology.orchestra.run.smithi037.stdout: "is_paused": false 2024-09-20T22:23:43.751 INFO:teuthology.orchestra.run.smithi037.stdout:} 2024-09-20T22:23:43.841 INFO:teuthology.orchestra.run.smithi037.stdout: 5 156960 11.24 MB/sec execute 241 sec latency 60.331 ms 2024-09-20T22:23:43.929 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:43 smithi194 ceph-mon[51725]: from='client.34180 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:23:43.929 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:43 smithi194 ceph-mon[51725]: from='client.? 172.21.15.37:0/1358906011' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:23:44.475 INFO:teuthology.orchestra.run.smithi037.stdout:HEALTH_OK 2024-09-20T22:23:44.744 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:44 smithi037 ceph-mon[65640]: from='client.44133 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:23:44.744 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:44 smithi037 ceph-mon[65640]: pgmap v44: 97 pgs: 97 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 9.0 MiB/s wr, 880 op/s 2024-09-20T22:23:44.744 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:44 smithi037 ceph-mon[65640]: from='client.? 172.21.15.37:0/269332205' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T22:23:44.841 INFO:teuthology.orchestra.run.smithi037.stdout: 5 157476 11.21 MB/sec execute 242 sec latency 56.197 ms 2024-09-20T22:23:44.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:44 smithi194 ceph-mon[51725]: from='client.44133 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:23:44.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:44 smithi194 ceph-mon[51725]: pgmap v44: 97 pgs: 97 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 9.0 MiB/s wr, 880 op/s 2024-09-20T22:23:44.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:44 smithi194 ceph-mon[51725]: from='client.? 172.21.15.37:0/269332205' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T22:23:45.841 INFO:teuthology.orchestra.run.smithi037.stdout: 5 157796 11.20 MB/sec execute 243 sec latency 34.395 ms 2024-09-20T22:23:46.841 INFO:teuthology.orchestra.run.smithi037.stdout: 5 158264 11.23 MB/sec execute 244 sec latency 44.507 ms 2024-09-20T22:23:47.016 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:46 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:47.016 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:46 smithi037 ceph-mon[65640]: pgmap v45: 97 pgs: 97 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 6.2 MiB/s wr, 575 op/s 2024-09-20T22:23:47.017 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:46 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:47.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:46 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:47.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:46 smithi194 ceph-mon[51725]: pgmap v45: 97 pgs: 97 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 6.2 MiB/s wr, 575 op/s 2024-09-20T22:23:47.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:46 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:47.841 INFO:teuthology.orchestra.run.smithi037.stdout: 5 158720 11.22 MB/sec execute 245 sec latency 60.567 ms 2024-09-20T22:23:48.241 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:48 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:48.241 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:48 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:48.241 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:48 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:48.241 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:48 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:48.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:48 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:48.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:48 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:48.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:48 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:48.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:48 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:48.841 INFO:teuthology.orchestra.run.smithi037.stdout: 5 159026 11.21 MB/sec execute 246 sec latency 41.457 ms 2024-09-20T22:23:49.298 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:49 smithi037 ceph-mon[65640]: pgmap v46: 97 pgs: 97 active+clean; 330 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 7.4 MiB/s wr, 684 op/s 2024-09-20T22:23:49.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:49 smithi194 ceph-mon[51725]: pgmap v46: 97 pgs: 97 active+clean; 330 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 7.4 MiB/s wr, 684 op/s 2024-09-20T22:23:49.841 INFO:teuthology.orchestra.run.smithi037.stdout: 5 159325 11.21 MB/sec execute 247 sec latency 41.935 ms 2024-09-20T22:23:50.841 INFO:teuthology.orchestra.run.smithi037.stdout: 5 159694 11.21 MB/sec execute 248 sec latency 46.753 ms 2024-09-20T22:23:51.088 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:51 smithi037 ceph-mon[65640]: pgmap v47: 97 pgs: 97 active+clean; 327 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 6.4 MiB/s wr, 586 op/s 2024-09-20T22:23:51.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:51 smithi194 ceph-mon[51725]: pgmap v47: 97 pgs: 97 active+clean; 327 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 6.4 MiB/s wr, 586 op/s 2024-09-20T22:23:51.841 INFO:teuthology.orchestra.run.smithi037.stdout: 5 160025 11.20 MB/sec execute 249 sec latency 42.866 ms 2024-09-20T22:23:52.103 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:52 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:23:52.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:52 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:23:52.842 INFO:teuthology.orchestra.run.smithi037.stdout: 5 160262 11.19 MB/sec execute 250 sec latency 44.115 ms 2024-09-20T22:23:53.121 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:53 smithi037 ceph-mon[65640]: pgmap v48: 97 pgs: 97 active+clean; 327 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.3 MiB/s wr, 537 op/s 2024-09-20T22:23:53.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:53 smithi194 ceph-mon[51725]: pgmap v48: 97 pgs: 97 active+clean; 327 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.3 MiB/s wr, 537 op/s 2024-09-20T22:23:53.842 INFO:teuthology.orchestra.run.smithi037.stdout: 5 160892 11.18 MB/sec execute 251 sec latency 28.705 ms 2024-09-20T22:23:54.842 INFO:teuthology.orchestra.run.smithi037.stdout: 5 161451 11.20 MB/sec execute 252 sec latency 23.579 ms 2024-09-20T22:23:55.143 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:55 smithi037 ceph-mon[65640]: pgmap v49: 97 pgs: 97 active+clean; 332 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.6 MiB/s wr, 736 op/s 2024-09-20T22:23:55.143 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:55 smithi037 ceph-mon[65640]: mgrmap e44: smithi037.lykblh(active, since 92s), standbys: smithi194.bkcglt 2024-09-20T22:23:55.143 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:55 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:55.143 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:55 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:55.143 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:55 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:23:55.143 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:55 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:23:55.143 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:55 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:55.144 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:55 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:55.144 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:55 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:23:55.144 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:55 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:23:55.144 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:55 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:23:55.144 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:55 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:23:55.144 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:55 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:55.144 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:55 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi037"}]: dispatch 2024-09-20T22:23:55.144 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:55 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi037"}]': finished 2024-09-20T22:23:55.144 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:55 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi194"}]: dispatch 2024-09-20T22:23:55.144 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:55 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi194"}]': finished 2024-09-20T22:23:55.144 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:55 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-20T22:23:55.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:55 smithi194 ceph-mon[51725]: pgmap v49: 97 pgs: 97 active+clean; 332 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.6 MiB/s wr, 736 op/s 2024-09-20T22:23:55.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:55 smithi194 ceph-mon[51725]: mgrmap e44: smithi037.lykblh(active, since 92s), standbys: smithi194.bkcglt 2024-09-20T22:23:55.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:55 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:55.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:55 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:55.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:55 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:23:55.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:55 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:23:55.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:55 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:55.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:55 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:55.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:55 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:23:55.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:55 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:23:55.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:55 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:23:55.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:55 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:23:55.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:55 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:55.346 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:55 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi037"}]: dispatch 2024-09-20T22:23:55.346 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:55 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi037"}]': finished 2024-09-20T22:23:55.346 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:55 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi194"}]: dispatch 2024-09-20T22:23:55.346 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:55 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi194"}]': finished 2024-09-20T22:23:55.346 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:55 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-20T22:23:55.842 INFO:teuthology.orchestra.run.smithi037.stdout: 5 162262 11.24 MB/sec execute 253 sec latency 25.604 ms 2024-09-20T22:23:56.149 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:56 smithi037 ceph-mon[65640]: Upgrade: Setting container_image for all crash 2024-09-20T22:23:56.149 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:56 smithi037 ceph-mon[65640]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-20T22:23:56.149 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:56 smithi037 ceph-mon[65640]: Upgrade: osd.0 is safe to restart 2024-09-20T22:23:56.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:56 smithi194 ceph-mon[51725]: Upgrade: Setting container_image for all crash 2024-09-20T22:23:56.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:56 smithi194 ceph-mon[51725]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-20T22:23:56.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:56 smithi194 ceph-mon[51725]: Upgrade: osd.0 is safe to restart 2024-09-20T22:23:56.842 INFO:teuthology.orchestra.run.smithi037.stdout: 5 162903 11.28 MB/sec execute 254 sec latency 26.783 ms 2024-09-20T22:23:57.159 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:57 smithi037 ceph-mon[65640]: pgmap v50: 97 pgs: 97 active+clean; 332 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 4.5 MiB/s wr, 455 op/s 2024-09-20T22:23:57.495 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:57 smithi194 ceph-mon[51725]: pgmap v50: 97 pgs: 97 active+clean; 332 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 4.5 MiB/s wr, 455 op/s 2024-09-20T22:23:57.842 INFO:teuthology.orchestra.run.smithi037.stdout: 5 163570 11.32 MB/sec execute 255 sec latency 34.318 ms 2024-09-20T22:23:58.842 INFO:teuthology.orchestra.run.smithi037.stdout: 5 163902 11.32 MB/sec execute 256 sec latency 52.352 ms 2024-09-20T22:23:59.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:59 smithi194 ceph-mon[51725]: pgmap v51: 97 pgs: 97 active+clean; 338 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.3 MiB/s wr, 784 op/s 2024-09-20T22:23:59.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:59 smithi194 ceph-mon[51725]: Upgrade: Updating osd.0 2024-09-20T22:23:59.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:59 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:59.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:59 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-20T22:23:59.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:59 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:23:59.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:23:59 smithi194 ceph-mon[51725]: Deploying daemon osd.0 on smithi037 2024-09-20T22:23:59.187 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:59 smithi037 ceph-mon[65640]: pgmap v51: 97 pgs: 97 active+clean; 338 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.3 MiB/s wr, 784 op/s 2024-09-20T22:23:59.187 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:59 smithi037 ceph-mon[65640]: Upgrade: Updating osd.0 2024-09-20T22:23:59.187 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:59 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:23:59.187 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:59 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-20T22:23:59.187 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:59 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:23:59.187 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:23:59 smithi037 ceph-mon[65640]: Deploying daemon osd.0 on smithi037 2024-09-20T22:23:59.842 INFO:teuthology.orchestra.run.smithi037.stdout: 5 164257 11.29 MB/sec execute 257 sec latency 64.041 ms 2024-09-20T22:24:00.842 INFO:teuthology.orchestra.run.smithi037.stdout: 5 164591 11.26 MB/sec execute 258 sec latency 53.301 ms 2024-09-20T22:24:01.233 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:01 smithi037 ceph-mon[65640]: pgmap v52: 97 pgs: 97 active+clean; 341 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 7.3 MiB/s wr, 728 op/s 2024-09-20T22:24:01.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:01 smithi194 ceph-mon[51725]: pgmap v52: 97 pgs: 97 active+clean; 341 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 7.3 MiB/s wr, 728 op/s 2024-09-20T22:24:01.842 INFO:teuthology.orchestra.run.smithi037.stdout: 5 164815 11.24 MB/sec execute 259 sec latency 40.021 ms 2024-09-20T22:24:02.842 INFO:teuthology.orchestra.run.smithi037.stdout: 5 165179 11.26 MB/sec execute 260 sec latency 42.234 ms 2024-09-20T22:24:03.279 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:03 smithi037 ceph-mon[65640]: pgmap v53: 97 pgs: 97 active+clean; 341 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 6.3 MiB/s wr, 580 op/s 2024-09-20T22:24:03.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:03 smithi194 ceph-mon[51725]: pgmap v53: 97 pgs: 97 active+clean; 341 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 6.3 MiB/s wr, 580 op/s 2024-09-20T22:24:03.786 INFO:journalctl@ceph.osd.0.smithi037.stdout:Sep 20 22:24:03 smithi037 systemd[1]: Stopping Ceph osd.0 for f9a6357a-779c-11ef-baf5-efdc52797490... 2024-09-20T22:24:03.842 INFO:teuthology.orchestra.run.smithi037.stdout: 5 165538 11.24 MB/sec execute 261 sec latency 56.457 ms 2024-09-20T22:24:04.288 INFO:journalctl@ceph.osd.0.smithi037.stdout:Sep 20 22:24:04 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-0[34232]: 2024-09-20T22:24:04.242+0000 7fef70963700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-20T22:24:04.288 INFO:journalctl@ceph.osd.0.smithi037.stdout:Sep 20 22:24:04 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-0[34232]: 2024-09-20T22:24:04.242+0000 7fef70963700 -1 osd.0 73 *** Got signal Terminated *** 2024-09-20T22:24:04.288 INFO:journalctl@ceph.osd.0.smithi037.stdout:Sep 20 22:24:04 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-0[34232]: 2024-09-20T22:24:04.242+0000 7fef70963700 -1 osd.0 73 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-20T22:24:04.843 INFO:teuthology.orchestra.run.smithi037.stdout: 5 165910 11.24 MB/sec execute 262 sec latency 62.760 ms 2024-09-20T22:24:05.060 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:05 smithi037 ceph-mon[65640]: pgmap v54: 97 pgs: 97 active+clean; 326 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.3 MiB/s wr, 795 op/s 2024-09-20T22:24:05.061 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:05 smithi037 ceph-mon[65640]: osd.0 marked itself down and dead 2024-09-20T22:24:05.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:05 smithi194 ceph-mon[51725]: pgmap v54: 97 pgs: 97 active+clean; 326 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.3 MiB/s wr, 795 op/s 2024-09-20T22:24:05.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:05 smithi194 ceph-mon[51725]: osd.0 marked itself down and dead 2024-09-20T22:24:05.842 INFO:teuthology.orchestra.run.smithi037.stdout: 5 166062 11.21 MB/sec execute 263 sec latency 780.778 ms 2024-09-20T22:24:05.853 INFO:journalctl@ceph.osd.0.smithi037.stdout:Sep 20 22:24:05 smithi037 podman[71035]: 2024-09-20 22:24:05.737382647 +0000 UTC m=+1.766199782 container died e54e84fd5030c0e68a2ba0607a696e4dbf6b1a91e6ebb76e7a9789db74c577af (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-0, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, distribution-scope=public, version=8, ceph=True, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container) 2024-09-20T22:24:06.109 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:06 smithi037 ceph-mon[65640]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-20T22:24:06.110 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:06 smithi037 ceph-mon[65640]: osdmap e74: 8 total, 7 up, 8 in 2024-09-20T22:24:06.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:06 smithi194 ceph-mon[51725]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-20T22:24:06.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:06 smithi194 ceph-mon[51725]: osdmap e74: 8 total, 7 up, 8 in 2024-09-20T22:24:06.616 INFO:journalctl@ceph.osd.0.smithi037.stdout:Sep 20 22:24:06 smithi037 podman[71035]: 2024-09-20 22:24:06.472658725 +0000 UTC m=+2.501475857 container cleanup e54e84fd5030c0e68a2ba0607a696e4dbf6b1a91e6ebb76e7a9789db74c577af (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-0, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, version=8, io.openshift.expose-services=, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, release=754, ceph=True, name=centos-stream, vcs-type=git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, io.openshift.tags=base centos centos-stream, distribution-scope=public, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD) 2024-09-20T22:24:06.616 INFO:journalctl@ceph.osd.0.smithi037.stdout:Sep 20 22:24:06 smithi037 bash[71035]: ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-0 2024-09-20T22:24:06.843 INFO:teuthology.orchestra.run.smithi037.stdout: 5 166336 11.19 MB/sec execute 264 sec latency 1025.142 ms 2024-09-20T22:24:06.866 INFO:journalctl@ceph.osd.0.smithi037.stdout:Sep 20 22:24:06 smithi037 podman[71052]: 2024-09-20 22:24:06.783084542 +0000 UTC m=+1.046074694 container remove e54e84fd5030c0e68a2ba0607a696e4dbf6b1a91e6ebb76e7a9789db74c577af (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-0, RELEASE=HEAD, ceph=True, architecture=x86_64, GIT_CLEAN=True, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, version=8) 2024-09-20T22:24:07.117 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:07 smithi037 ceph-mon[65640]: pgmap v56: 97 pgs: 13 stale+active+clean, 84 active+clean; 326 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.1 MiB/s wr, 716 op/s 2024-09-20T22:24:07.118 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:07 smithi037 ceph-mon[65640]: osdmap e75: 8 total, 7 up, 8 in 2024-09-20T22:24:07.118 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:07 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:24:07.118 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:07 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:24:07.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:07 smithi194 ceph-mon[51725]: pgmap v56: 97 pgs: 13 stale+active+clean, 84 active+clean; 326 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.1 MiB/s wr, 716 op/s 2024-09-20T22:24:07.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:07 smithi194 ceph-mon[51725]: osdmap e75: 8 total, 7 up, 8 in 2024-09-20T22:24:07.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:07 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:24:07.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:07 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:24:07.381 INFO:journalctl@ceph.osd.0.smithi037.stdout:Sep 20 22:24:07 smithi037 podman[71135]: 2024-09-20 22:24:07.238974058 +0000 UTC m=+0.024917206 image pull cc9251b744a3faafca7c3095f79bfff81873afe823f31a03a11e015a34abcdbd quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5 2024-09-20T22:24:07.499 INFO:journalctl@ceph.osd.0.smithi037.stdout:Sep 20 22:24:07 smithi037 podman[71135]: 2024-09-20 22:24:07.472481493 +0000 UTC m=+0.258424622 container create 3ba5c3fef458ec6b876146afe64832851f593a0e1b13430f9fe8385a4722f145 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-0-deactivate, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-9e2ac70, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731) 2024-09-20T22:24:07.843 INFO:teuthology.orchestra.run.smithi037.stdout: 5 166631 11.18 MB/sec execute 265 sec latency 43.873 ms 2024-09-20T22:24:07.896 INFO:journalctl@ceph.osd.0.smithi037.stdout:Sep 20 22:24:07 smithi037 podman[71135]: 2024-09-20 22:24:07.851141469 +0000 UTC m=+0.637084612 container init 3ba5c3fef458ec6b876146afe64832851f593a0e1b13430f9fe8385a4722f145 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-0-deactivate, RELEASE=squid-9e2ac70, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=) 2024-09-20T22:24:07.897 INFO:journalctl@ceph.osd.0.smithi037.stdout:Sep 20 22:24:07 smithi037 podman[71135]: 2024-09-20 22:24:07.860006085 +0000 UTC m=+0.645949223 container start 3ba5c3fef458ec6b876146afe64832851f593a0e1b13430f9fe8385a4722f145 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-0-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, RELEASE=squid-9e2ac70, GIT_CLEAN=True, io.buildah.version=1.37.2) 2024-09-20T22:24:08.148 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:08 smithi037 ceph-mon[65640]: Health check failed: Degraded data redundancy: 132/1530 objects degraded (8.627%), 20 pgs degraded (PG_DEGRADED) 2024-09-20T22:24:08.148 INFO:journalctl@ceph.osd.0.smithi037.stdout:Sep 20 22:24:07 smithi037 podman[71135]: 2024-09-20 22:24:07.993241841 +0000 UTC m=+0.779184972 container attach 3ba5c3fef458ec6b876146afe64832851f593a0e1b13430f9fe8385a4722f145 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-0-deactivate, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, RELEASE=squid-9e2ac70, GIT_BRANCH=HEAD) 2024-09-20T22:24:08.409 INFO:journalctl@ceph.osd.0.smithi037.stdout:Sep 20 22:24:08 smithi037 podman[71135]: 2024-09-20 22:24:08.226303971 +0000 UTC m=+1.012247118 container died 3ba5c3fef458ec6b876146afe64832851f593a0e1b13430f9fe8385a4722f145 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-0-deactivate, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=squid-9e2ac70, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-09-20T22:24:08.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:08 smithi194 ceph-mon[51725]: Health check failed: Degraded data redundancy: 132/1530 objects degraded (8.627%), 20 pgs degraded (PG_DEGRADED) 2024-09-20T22:24:08.843 INFO:teuthology.orchestra.run.smithi037.stdout: 5 167005 11.20 MB/sec execute 266 sec latency 36.798 ms 2024-09-20T22:24:09.186 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:09 smithi037 ceph-mon[65640]: pgmap v58: 97 pgs: 5 active+undersized, 4 stale+active+clean, 20 active+undersized+degraded, 68 active+clean; 327 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.4 KiB/s rd, 4.7 MiB/s wr, 472 op/s; 132/1530 objects degraded (8.627%) 2024-09-20T22:24:09.445 INFO:journalctl@ceph.osd.0.smithi037.stdout:Sep 20 22:24:09 smithi037 podman[71157]: 2024-09-20 22:24:09.217189923 +0000 UTC m=+0.978530485 container remove 3ba5c3fef458ec6b876146afe64832851f593a0e1b13430f9fe8385a4722f145 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-0-deactivate, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, RELEASE=squid-9e2ac70, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0) 2024-09-20T22:24:09.445 INFO:journalctl@ceph.osd.0.smithi037.stdout:Sep 20 22:24:09 smithi037 systemd[1]: ceph-f9a6357a-779c-11ef-baf5-efdc52797490@osd.0.service: Deactivated successfully. 2024-09-20T22:24:09.445 INFO:journalctl@ceph.osd.0.smithi037.stdout:Sep 20 22:24:09 smithi037 systemd[1]: Stopped Ceph osd.0 for f9a6357a-779c-11ef-baf5-efdc52797490. 2024-09-20T22:24:09.445 INFO:journalctl@ceph.osd.0.smithi037.stdout:Sep 20 22:24:09 smithi037 systemd[1]: ceph-f9a6357a-779c-11ef-baf5-efdc52797490@osd.0.service: Consumed 2min 53.519s CPU time. 2024-09-20T22:24:09.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:09 smithi194 ceph-mon[51725]: pgmap v58: 97 pgs: 5 active+undersized, 4 stale+active+clean, 20 active+undersized+degraded, 68 active+clean; 327 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.4 KiB/s rd, 4.7 MiB/s wr, 472 op/s; 132/1530 objects degraded (8.627%) 2024-09-20T22:24:09.710 INFO:journalctl@ceph.osd.0.smithi037.stdout:Sep 20 22:24:09 smithi037 systemd[1]: Starting Ceph osd.0 for f9a6357a-779c-11ef-baf5-efdc52797490... 2024-09-20T22:24:09.843 INFO:teuthology.orchestra.run.smithi037.stdout: 5 167283 11.18 MB/sec execute 267 sec latency 67.043 ms 2024-09-20T22:24:09.967 INFO:journalctl@ceph.osd.0.smithi037.stdout:Sep 20 22:24:09 smithi037 podman[71252]: 2024-09-20 22:24:09.742831311 +0000 UTC m=+0.023279203 image pull cc9251b744a3faafca7c3095f79bfff81873afe823f31a03a11e015a34abcdbd quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5 2024-09-20T22:24:09.967 INFO:journalctl@ceph.osd.0.smithi037.stdout:Sep 20 22:24:09 smithi037 podman[71252]: 2024-09-20 22:24:09.935603569 +0000 UTC m=+0.216051449 container create 7c465a39f5ad2c1d151a8aa4eb0a294a30008841a49b1d879a0e66ee924115f2 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-0-activate, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, RELEASE=squid-9e2ac70, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-20T22:24:10.471 INFO:journalctl@ceph.osd.0.smithi037.stdout:Sep 20 22:24:10 smithi037 podman[71252]: 2024-09-20 22:24:10.390110024 +0000 UTC m=+0.670557923 container init 7c465a39f5ad2c1d151a8aa4eb0a294a30008841a49b1d879a0e66ee924115f2 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-0-activate, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=squid-9e2ac70, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2) 2024-09-20T22:24:10.472 INFO:journalctl@ceph.osd.0.smithi037.stdout:Sep 20 22:24:10 smithi037 podman[71252]: 2024-09-20 22:24:10.398450557 +0000 UTC m=+0.678898442 container start 7c465a39f5ad2c1d151a8aa4eb0a294a30008841a49b1d879a0e66ee924115f2 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-0-activate, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, RELEASE=squid-9e2ac70, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0) 2024-09-20T22:24:10.731 INFO:journalctl@ceph.osd.0.smithi037.stdout:Sep 20 22:24:10 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-0-activate[71267]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T22:24:10.731 INFO:journalctl@ceph.osd.0.smithi037.stdout:Sep 20 22:24:10 smithi037 bash[71252]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T22:24:10.731 INFO:journalctl@ceph.osd.0.smithi037.stdout:Sep 20 22:24:10 smithi037 podman[71252]: 2024-09-20 22:24:10.573888821 +0000 UTC m=+0.854336745 container attach 7c465a39f5ad2c1d151a8aa4eb0a294a30008841a49b1d879a0e66ee924115f2 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-0-activate, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, RELEASE=squid-9e2ac70, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS) 2024-09-20T22:24:10.731 INFO:journalctl@ceph.osd.0.smithi037.stdout:Sep 20 22:24:10 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-0-activate[71267]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T22:24:10.731 INFO:journalctl@ceph.osd.0.smithi037.stdout:Sep 20 22:24:10 smithi037 bash[71252]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T22:24:10.843 INFO:teuthology.orchestra.run.smithi037.stdout: 5 167574 11.17 MB/sec execute 268 sec latency 68.126 ms 2024-09-20T22:24:11.232 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:11 smithi037 ceph-mon[65640]: pgmap v59: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 331 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.6 KiB/s rd, 5.7 MiB/s wr, 586 op/s; 196/1560 objects degraded (12.564%) 2024-09-20T22:24:11.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:11 smithi194 ceph-mon[51725]: pgmap v59: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 331 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.6 KiB/s rd, 5.7 MiB/s wr, 586 op/s; 196/1560 objects degraded (12.564%) 2024-09-20T22:24:11.843 INFO:teuthology.orchestra.run.smithi037.stdout: 5 167989 11.16 MB/sec execute 269 sec latency 35.426 ms 2024-09-20T22:24:12.110 INFO:journalctl@ceph.osd.0.smithi037.stdout:Sep 20 22:24:12 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-0-activate[71267]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-20T22:24:12.110 INFO:journalctl@ceph.osd.0.smithi037.stdout:Sep 20 22:24:12 smithi037 bash[71252]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-20T22:24:12.110 INFO:journalctl@ceph.osd.0.smithi037.stdout:Sep 20 22:24:12 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-0-activate[71267]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-20T22:24:12.110 INFO:journalctl@ceph.osd.0.smithi037.stdout:Sep 20 22:24:12 smithi037 bash[71252]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-20T22:24:12.621 INFO:journalctl@ceph.osd.0.smithi037.stdout:Sep 20 22:24:12 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-0-activate[71267]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-20T22:24:12.621 INFO:journalctl@ceph.osd.0.smithi037.stdout:Sep 20 22:24:12 smithi037 bash[71252]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-20T22:24:12.621 INFO:journalctl@ceph.osd.0.smithi037.stdout:Sep 20 22:24:12 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-0-activate[71267]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-20T22:24:12.621 INFO:journalctl@ceph.osd.0.smithi037.stdout:Sep 20 22:24:12 smithi037 bash[71252]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-20T22:24:12.621 INFO:journalctl@ceph.osd.0.smithi037.stdout:Sep 20 22:24:12 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-0-activate[71267]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-20T22:24:12.621 INFO:journalctl@ceph.osd.0.smithi037.stdout:Sep 20 22:24:12 smithi037 bash[71252]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-20T22:24:12.622 INFO:journalctl@ceph.osd.0.smithi037.stdout:Sep 20 22:24:12 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-0-activate[71267]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-20T22:24:12.622 INFO:journalctl@ceph.osd.0.smithi037.stdout:Sep 20 22:24:12 smithi037 bash[71252]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-20T22:24:12.622 INFO:journalctl@ceph.osd.0.smithi037.stdout:Sep 20 22:24:12 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-0-activate[71267]: --> ceph-volume raw activate successful for osd ID: 0 2024-09-20T22:24:12.622 INFO:journalctl@ceph.osd.0.smithi037.stdout:Sep 20 22:24:12 smithi037 bash[71252]: --> ceph-volume raw activate successful for osd ID: 0 2024-09-20T22:24:12.622 INFO:journalctl@ceph.osd.0.smithi037.stdout:Sep 20 22:24:12 smithi037 podman[71252]: 2024-09-20 22:24:12.516813381 +0000 UTC m=+2.797261272 container died 7c465a39f5ad2c1d151a8aa4eb0a294a30008841a49b1d879a0e66ee924115f2 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-0-activate, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=squid-9e2ac70, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-20T22:24:12.843 INFO:teuthology.orchestra.run.smithi037.stdout: 5 168265 11.14 MB/sec execute 270 sec latency 48.039 ms 2024-09-20T22:24:13.160 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:13 smithi037 ceph-mon[65640]: pgmap v60: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 331 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 2.9 KiB/s rd, 2.7 MiB/s wr, 263 op/s; 196/1560 objects degraded (12.564%) 2024-09-20T22:24:13.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:13 smithi194 ceph-mon[51725]: pgmap v60: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 331 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 2.9 KiB/s rd, 2.7 MiB/s wr, 263 op/s; 196/1560 objects degraded (12.564%) 2024-09-20T22:24:13.843 INFO:teuthology.orchestra.run.smithi037.stdout: 5 168498 11.11 MB/sec execute 271 sec latency 52.503 ms 2024-09-20T22:24:13.939 INFO:journalctl@ceph.osd.0.smithi037.stdout:Sep 20 22:24:13 smithi037 podman[71416]: 2024-09-20 22:24:13.766391118 +0000 UTC m=+1.240153042 container remove 7c465a39f5ad2c1d151a8aa4eb0a294a30008841a49b1d879a0e66ee924115f2 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-0-activate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-9e2ac70, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-20T22:24:14.200 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:14 smithi037 ceph-mon[65640]: Health check update: Degraded data redundancy: 220/1665 objects degraded (13.213%), 29 pgs degraded (PG_DEGRADED) 2024-09-20T22:24:14.201 INFO:journalctl@ceph.osd.0.smithi037.stdout:Sep 20 22:24:14 smithi037 podman[71505]: 2024-09-20 22:24:13.915109791 +0000 UTC m=+0.023929055 image pull cc9251b744a3faafca7c3095f79bfff81873afe823f31a03a11e015a34abcdbd quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5 2024-09-20T22:24:14.201 INFO:journalctl@ceph.osd.0.smithi037.stdout:Sep 20 22:24:14 smithi037 podman[71505]: 2024-09-20 22:24:14.10173398 +0000 UTC m=+0.210553230 container create c5568bd43c075314ec5f7e5f230f8a40319bd6033ce439b6ff40421150c50a20 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-0, RELEASE=squid-9e2ac70, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-09-20T22:24:14.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:14 smithi194 ceph-mon[51725]: Health check update: Degraded data redundancy: 220/1665 objects degraded (13.213%), 29 pgs degraded (PG_DEGRADED) 2024-09-20T22:24:14.705 INFO:journalctl@ceph.osd.0.smithi037.stdout:Sep 20 22:24:14 smithi037 podman[71505]: 2024-09-20 22:24:14.480027036 +0000 UTC m=+0.588846296 container init c5568bd43c075314ec5f7e5f230f8a40319bd6033ce439b6ff40421150c50a20 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-0, RELEASE=squid-9e2ac70, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-20T22:24:14.705 INFO:journalctl@ceph.osd.0.smithi037.stdout:Sep 20 22:24:14 smithi037 podman[71505]: 2024-09-20 22:24:14.493891251 +0000 UTC m=+0.602710504 container start c5568bd43c075314ec5f7e5f230f8a40319bd6033ce439b6ff40421150c50a20 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-0, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-9e2ac70, org.label-schema.license=GPLv2, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-09-20T22:24:14.705 INFO:journalctl@ceph.osd.0.smithi037.stdout:Sep 20 22:24:14 smithi037 ceph-osd[71522]: -- 172.21.15.37:0/1443711881 <== mon.1 v2:172.21.15.194:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x56107b8ce1e0 con 0x56107b8a2000 2024-09-20T22:24:14.705 INFO:journalctl@ceph.osd.0.smithi037.stdout:Sep 20 22:24:14 smithi037 bash[71505]: c5568bd43c075314ec5f7e5f230f8a40319bd6033ce439b6ff40421150c50a20 2024-09-20T22:24:14.705 INFO:journalctl@ceph.osd.0.smithi037.stdout:Sep 20 22:24:14 smithi037 systemd[1]: Started Ceph osd.0 for f9a6357a-779c-11ef-baf5-efdc52797490. 2024-09-20T22:24:14.843 INFO:teuthology.orchestra.run.smithi037.stdout: 5 168794 11.12 MB/sec execute 272 sec latency 48.304 ms 2024-09-20T22:24:15.124 INFO:teuthology.orchestra.run.smithi037.stdout:true 2024-09-20T22:24:15.254 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:15 smithi037 ceph-mon[65640]: pgmap v61: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 332 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 5.2 MiB/s wr, 506 op/s; 220/1665 objects degraded (13.213%) 2024-09-20T22:24:15.255 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:15 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:24:15.255 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:15 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:24:15.255 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:15 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:24:15.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:15 smithi194 ceph-mon[51725]: pgmap v61: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 332 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 5.2 MiB/s wr, 506 op/s; 220/1665 objects degraded (13.213%) 2024-09-20T22:24:15.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:15 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:24:15.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:15 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:24:15.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:15 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:24:15.843 INFO:teuthology.orchestra.run.smithi037.stdout: 5 169048 11.10 MB/sec execute 273 sec latency 63.470 ms 2024-09-20T22:24:16.055 INFO:teuthology.orchestra.run.smithi037.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T22:24:16.055 INFO:teuthology.orchestra.run.smithi037.stdout:alertmanager.smithi037 smithi037 *:9093,9094 running (4m) 28s ago 13m 27.9M - 0.25.0 c8568f914cd2 d1f99b3cd707 2024-09-20T22:24:16.055 INFO:teuthology.orchestra.run.smithi037.stdout:crash.smithi037 smithi037 running (44s) 28s ago 12m 6895k - 19.1.1-367-g9e2ac70b cc9251b744a3 77e510d8bdb1 2024-09-20T22:24:16.055 INFO:teuthology.orchestra.run.smithi037.stdout:crash.smithi194 smithi194 running (38s) 34s ago 12m 6895k - 19.1.1-367-g9e2ac70b cc9251b744a3 4d169d1309bc 2024-09-20T22:24:16.055 INFO:teuthology.orchestra.run.smithi037.stdout:grafana.smithi037 smithi037 *:3000 running (4m) 28s ago 12m 52.0M - dad864ee21e9 2306f3c2da19 2024-09-20T22:24:16.055 INFO:teuthology.orchestra.run.smithi037.stdout:mds.foofs.smithi037.sdtedi smithi037 running (7m) 28s ago 7m 37.9M - 17.2.0 e1d6a67b021e 0a7898d25b8d 2024-09-20T22:24:16.055 INFO:teuthology.orchestra.run.smithi037.stdout:mds.foofs.smithi194.lkroso smithi194 running (7m) 34s ago 7m 15.6M - 17.2.0 e1d6a67b021e 17736de30008 2024-09-20T22:24:16.055 INFO:teuthology.orchestra.run.smithi037.stdout:mgr.smithi037.lykblh smithi037 *:8443,9283,8765 running (3m) 28s ago 14m 495M - 19.1.1-367-g9e2ac70b cc9251b744a3 181e46fd9ba6 2024-09-20T22:24:16.055 INFO:teuthology.orchestra.run.smithi037.stdout:mgr.smithi194.bkcglt smithi194 *:8443,9283,8765 running (2m) 34s ago 12m 439M - 19.1.1-367-g9e2ac70b cc9251b744a3 1fa610c97517 2024-09-20T22:24:16.055 INFO:teuthology.orchestra.run.smithi037.stdout:mon.smithi037 smithi037 running (2m) 28s ago 14m 43.7M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 c0229eceb527 2024-09-20T22:24:16.056 INFO:teuthology.orchestra.run.smithi037.stdout:mon.smithi194 smithi194 running (96s) 34s ago 12m 37.2M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 97854807ba91 2024-09-20T22:24:16.056 INFO:teuthology.orchestra.run.smithi037.stdout:nfs.foo.0.0.smithi037.spabdc smithi037 *:2049 running (6m) 28s ago 6m 265M - 4.0 e1d6a67b021e 858ff9d558a2 2024-09-20T22:24:16.056 INFO:teuthology.orchestra.run.smithi037.stdout:nfs.foo.1.0.smithi194.qzjyej smithi194 *:2049 running (6m) 34s ago 6m 89.8M - 4.0 e1d6a67b021e c07fe0eb7c75 2024-09-20T22:24:16.056 INFO:teuthology.orchestra.run.smithi037.stdout:node-exporter.smithi037 smithi037 *:9100 running (3m) 28s ago 12m 16.0M - 1.5.0 0da6a335fe13 433e6aaf6137 2024-09-20T22:24:16.056 INFO:teuthology.orchestra.run.smithi037.stdout:node-exporter.smithi194 smithi194 *:9100 running (3m) 34s ago 12m 15.7M - 1.5.0 0da6a335fe13 1d30bb810ad1 2024-09-20T22:24:16.056 INFO:teuthology.orchestra.run.smithi037.stdout:osd.0 smithi037 starting - - - 1937M 2024-09-20T22:24:16.056 INFO:teuthology.orchestra.run.smithi037.stdout:osd.1 smithi037 running (11m) 28s ago 11m 482M 1937M 17.2.0 e1d6a67b021e 09647c133032 2024-09-20T22:24:16.056 INFO:teuthology.orchestra.run.smithi037.stdout:osd.2 smithi037 running (10m) 28s ago 10m 356M 1937M 17.2.0 e1d6a67b021e 6c15b1175256 2024-09-20T22:24:16.056 INFO:teuthology.orchestra.run.smithi037.stdout:osd.3 smithi037 running (10m) 28s ago 10m 493M 1937M 17.2.0 e1d6a67b021e 3a22952c6475 2024-09-20T22:24:16.056 INFO:teuthology.orchestra.run.smithi037.stdout:osd.4 smithi194 running (9m) 34s ago 9m 513M 2704M 17.2.0 e1d6a67b021e fa107be8915c 2024-09-20T22:24:16.056 INFO:teuthology.orchestra.run.smithi037.stdout:osd.5 smithi194 running (9m) 34s ago 9m 324M 2704M 17.2.0 e1d6a67b021e 290b21cbd131 2024-09-20T22:24:16.056 INFO:teuthology.orchestra.run.smithi037.stdout:osd.6 smithi194 running (8m) 34s ago 8m 511M 2704M 17.2.0 e1d6a67b021e 27f0720639ed 2024-09-20T22:24:16.056 INFO:teuthology.orchestra.run.smithi037.stdout:osd.7 smithi194 running (8m) 34s ago 8m 406M 2704M 17.2.0 e1d6a67b021e 62d163f4fd92 2024-09-20T22:24:16.056 INFO:teuthology.orchestra.run.smithi037.stdout:prometheus.smithi037 smithi037 *:9095 running (2m) 28s ago 11m 61.5M - 2.43.0 a07b618ecd1d 05b826dde2ff 2024-09-20T22:24:16.260 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:16 smithi037 ceph-mon[65640]: from='client.34196 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:24:16.511 INFO:journalctl@ceph.osd.0.smithi037.stdout:Sep 20 22:24:16 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-0[71518]: 2024-09-20T22:24:16.375+0000 7fb3bd739640 -1 Falling back to public interface 2024-09-20T22:24:16.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:16 smithi194 ceph-mon[51725]: from='client.34196 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:24:16.670 INFO:teuthology.orchestra.run.smithi037.stdout:{ 2024-09-20T22:24:16.670 INFO:teuthology.orchestra.run.smithi037.stdout: "mon": { 2024-09-20T22:24:16.671 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 2 2024-09-20T22:24:16.671 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:24:16.671 INFO:teuthology.orchestra.run.smithi037.stdout: "mgr": { 2024-09-20T22:24:16.671 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 2 2024-09-20T22:24:16.671 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:24:16.671 INFO:teuthology.orchestra.run.smithi037.stdout: "osd": { 2024-09-20T22:24:16.671 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7 2024-09-20T22:24:16.671 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:24:16.671 INFO:teuthology.orchestra.run.smithi037.stdout: "mds": { 2024-09-20T22:24:16.671 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T22:24:16.671 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:24:16.671 INFO:teuthology.orchestra.run.smithi037.stdout: "overall": { 2024-09-20T22:24:16.671 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 9, 2024-09-20T22:24:16.672 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 4 2024-09-20T22:24:16.672 INFO:teuthology.orchestra.run.smithi037.stdout: } 2024-09-20T22:24:16.672 INFO:teuthology.orchestra.run.smithi037.stdout:} 2024-09-20T22:24:16.843 INFO:teuthology.orchestra.run.smithi037.stdout: 5 169379 11.09 MB/sec execute 274 sec latency 69.160 ms 2024-09-20T22:24:17.250 INFO:teuthology.orchestra.run.smithi037.stdout:{ 2024-09-20T22:24:17.250 INFO:teuthology.orchestra.run.smithi037.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5", 2024-09-20T22:24:17.250 INFO:teuthology.orchestra.run.smithi037.stdout: "in_progress": true, 2024-09-20T22:24:17.251 INFO:teuthology.orchestra.run.smithi037.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-20T22:24:17.251 INFO:teuthology.orchestra.run.smithi037.stdout: "services_complete": [ 2024-09-20T22:24:17.251 INFO:teuthology.orchestra.run.smithi037.stdout: "mon", 2024-09-20T22:24:17.251 INFO:teuthology.orchestra.run.smithi037.stdout: "mgr", 2024-09-20T22:24:17.251 INFO:teuthology.orchestra.run.smithi037.stdout: "crash" 2024-09-20T22:24:17.251 INFO:teuthology.orchestra.run.smithi037.stdout: ], 2024-09-20T22:24:17.251 INFO:teuthology.orchestra.run.smithi037.stdout: "progress": "6/23 daemons upgraded", 2024-09-20T22:24:17.251 INFO:teuthology.orchestra.run.smithi037.stdout: "message": "Currently upgrading osd daemons", 2024-09-20T22:24:17.251 INFO:teuthology.orchestra.run.smithi037.stdout: "is_paused": false 2024-09-20T22:24:17.251 INFO:teuthology.orchestra.run.smithi037.stdout:} 2024-09-20T22:24:17.265 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:17 smithi037 ceph-mon[65640]: from='client.44145 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:24:17.266 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:17 smithi037 ceph-mon[65640]: pgmap v62: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 332 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.5 MiB/s wr, 445 op/s; 220/1665 objects degraded (13.213%) 2024-09-20T22:24:17.266 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:17 smithi037 ceph-mon[65640]: from='client.34204 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:24:17.266 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:17 smithi037 ceph-mon[65640]: from='client.? 172.21.15.37:0/3891753856' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:24:17.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:17 smithi194 ceph-mon[51725]: from='client.44145 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:24:17.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:17 smithi194 ceph-mon[51725]: pgmap v62: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 332 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.5 MiB/s wr, 445 op/s; 220/1665 objects degraded (13.213%) 2024-09-20T22:24:17.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:17 smithi194 ceph-mon[51725]: from='client.34204 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:24:17.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:17 smithi194 ceph-mon[51725]: from='client.? 172.21.15.37:0/3891753856' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:24:17.843 INFO:teuthology.orchestra.run.smithi037.stdout: 5 169770 11.07 MB/sec execute 275 sec latency 35.658 ms 2024-09-20T22:24:17.957 INFO:teuthology.orchestra.run.smithi037.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 220/1665 objects degraded (13.213%), 29 pgs degraded 2024-09-20T22:24:17.957 INFO:teuthology.orchestra.run.smithi037.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-20T22:24:17.957 INFO:teuthology.orchestra.run.smithi037.stdout: osd.0 (root=default,host=smithi037) is down 2024-09-20T22:24:17.957 INFO:teuthology.orchestra.run.smithi037.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 220/1665 objects degraded (13.213%), 29 pgs degraded 2024-09-20T22:24:17.957 INFO:teuthology.orchestra.run.smithi037.stdout: pg 1.0 is active+undersized+degraded, acting [7,6] 2024-09-20T22:24:17.958 INFO:teuthology.orchestra.run.smithi037.stdout: pg 2.0 is active+undersized+degraded, acting [7,1] 2024-09-20T22:24:17.958 INFO:teuthology.orchestra.run.smithi037.stdout: pg 2.1 is active+undersized+degraded, acting [2,3] 2024-09-20T22:24:17.958 INFO:teuthology.orchestra.run.smithi037.stdout: pg 2.4 is active+undersized+degraded, acting [1,7] 2024-09-20T22:24:17.958 INFO:teuthology.orchestra.run.smithi037.stdout: pg 2.5 is active+undersized+degraded, acting [7,4] 2024-09-20T22:24:17.958 INFO:teuthology.orchestra.run.smithi037.stdout: pg 2.8 is active+undersized+degraded, acting [7,1] 2024-09-20T22:24:17.958 INFO:teuthology.orchestra.run.smithi037.stdout: pg 2.c is active+undersized+degraded, acting [2,5] 2024-09-20T22:24:17.958 INFO:teuthology.orchestra.run.smithi037.stdout: pg 2.f is active+undersized+degraded, acting [4,7] 2024-09-20T22:24:17.958 INFO:teuthology.orchestra.run.smithi037.stdout: pg 2.10 is active+undersized+degraded, acting [2,1] 2024-09-20T22:24:17.958 INFO:teuthology.orchestra.run.smithi037.stdout: pg 2.13 is active+undersized+degraded, acting [4,3] 2024-09-20T22:24:17.958 INFO:teuthology.orchestra.run.smithi037.stdout: pg 2.15 is active+undersized+degraded, acting [1,5] 2024-09-20T22:24:17.958 INFO:teuthology.orchestra.run.smithi037.stdout: pg 2.19 is active+undersized+degraded, acting [3,6] 2024-09-20T22:24:17.958 INFO:teuthology.orchestra.run.smithi037.stdout: pg 2.1d is active+undersized+degraded, acting [7,6] 2024-09-20T22:24:17.958 INFO:teuthology.orchestra.run.smithi037.stdout: pg 2.1e is active+undersized+degraded, acting [3,5] 2024-09-20T22:24:17.958 INFO:teuthology.orchestra.run.smithi037.stdout: pg 2.1f is active+undersized+degraded, acting [2,4] 2024-09-20T22:24:17.958 INFO:teuthology.orchestra.run.smithi037.stdout: pg 3.1 is active+undersized+degraded, acting [4,3] 2024-09-20T22:24:17.958 INFO:teuthology.orchestra.run.smithi037.stdout: pg 3.3 is active+undersized+degraded, acting [4,6] 2024-09-20T22:24:17.959 INFO:teuthology.orchestra.run.smithi037.stdout: pg 3.6 is active+undersized+degraded, acting [1,4] 2024-09-20T22:24:17.959 INFO:teuthology.orchestra.run.smithi037.stdout: pg 3.7 is active+undersized+degraded, acting [3,7] 2024-09-20T22:24:17.959 INFO:teuthology.orchestra.run.smithi037.stdout: pg 3.b is active+undersized+degraded, acting [3,4] 2024-09-20T22:24:17.959 INFO:teuthology.orchestra.run.smithi037.stdout: pg 3.f is active+undersized+degraded, acting [7,4] 2024-09-20T22:24:17.959 INFO:teuthology.orchestra.run.smithi037.stdout: pg 3.10 is active+undersized+degraded, acting [6,5] 2024-09-20T22:24:17.959 INFO:teuthology.orchestra.run.smithi037.stdout: pg 3.12 is active+undersized+degraded, acting [7,3] 2024-09-20T22:24:17.959 INFO:teuthology.orchestra.run.smithi037.stdout: pg 3.17 is active+undersized+degraded, acting [5,3] 2024-09-20T22:24:17.959 INFO:teuthology.orchestra.run.smithi037.stdout: pg 3.18 is active+undersized+degraded, acting [3,1] 2024-09-20T22:24:17.959 INFO:teuthology.orchestra.run.smithi037.stdout: pg 3.1b is active+undersized+degraded, acting [4,7] 2024-09-20T22:24:17.959 INFO:teuthology.orchestra.run.smithi037.stdout: pg 3.1f is active+undersized+degraded, acting [5,2] 2024-09-20T22:24:17.959 INFO:teuthology.orchestra.run.smithi037.stdout: pg 4.0 is active+undersized+degraded, acting [3,7] 2024-09-20T22:24:17.959 INFO:teuthology.orchestra.run.smithi037.stdout: pg 4.19 is active+undersized+degraded, acting [3,2] 2024-09-20T22:24:18.277 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:18 smithi037 ceph-mon[65640]: from='client.34212 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:24:18.278 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:18 smithi037 ceph-mon[65640]: pgmap v63: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 327 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.9 MiB/s wr, 464 op/s; 212/1614 objects degraded (13.135%) 2024-09-20T22:24:18.278 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:18 smithi037 ceph-mon[65640]: from='client.? 172.21.15.37:0/79420174' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T22:24:18.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:18 smithi194 ceph-mon[51725]: from='client.34212 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:24:18.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:18 smithi194 ceph-mon[51725]: pgmap v63: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 327 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.9 MiB/s wr, 464 op/s; 212/1614 objects degraded (13.135%) 2024-09-20T22:24:18.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:18 smithi194 ceph-mon[51725]: from='client.? 172.21.15.37:0/79420174' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T22:24:18.843 INFO:teuthology.orchestra.run.smithi037.stdout: 5 170185 11.08 MB/sec execute 276 sec latency 35.649 ms 2024-09-20T22:24:19.554 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:19 smithi037 ceph-mon[65640]: Health check update: Degraded data redundancy: 212/1614 objects degraded (13.135%), 29 pgs degraded (PG_DEGRADED) 2024-09-20T22:24:19.843 INFO:teuthology.orchestra.run.smithi037.stdout: 5 170579 11.09 MB/sec execute 277 sec latency 53.680 ms 2024-09-20T22:24:19.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:19 smithi194 ceph-mon[51725]: Health check update: Degraded data redundancy: 212/1614 objects degraded (13.135%), 29 pgs degraded (PG_DEGRADED) 2024-09-20T22:24:20.640 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:20 smithi037 ceph-mon[65640]: pgmap v64: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 329 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.3 MiB/s wr, 455 op/s; 216/1614 objects degraded (13.383%) 2024-09-20T22:24:20.843 INFO:teuthology.orchestra.run.smithi037.stdout: 5 170669 11.06 MB/sec execute 278 sec latency 117.213 ms 2024-09-20T22:24:20.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:20 smithi194 ceph-mon[51725]: pgmap v64: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 329 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.3 MiB/s wr, 455 op/s; 216/1614 objects degraded (13.383%) 2024-09-20T22:24:21.843 INFO:teuthology.orchestra.run.smithi037.stdout: 5 170860 11.05 MB/sec execute 279 sec latency 93.570 ms 2024-09-20T22:24:22.844 INFO:teuthology.orchestra.run.smithi037.stdout: 5 170958 11.01 MB/sec execute 280 sec latency 207.928 ms 2024-09-20T22:24:23.046 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:22 smithi037 ceph-mon[65640]: pgmap v65: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 329 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.7 MiB/s wr, 379 op/s; 216/1614 objects degraded (13.383%) 2024-09-20T22:24:23.047 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:22 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:24:23.047 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:22 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:24:23.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:22 smithi194 ceph-mon[51725]: pgmap v65: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 329 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.7 MiB/s wr, 379 op/s; 216/1614 objects degraded (13.383%) 2024-09-20T22:24:23.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:22 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:24:23.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:22 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:24:23.572 INFO:journalctl@ceph.osd.0.smithi037.stdout:Sep 20 22:24:23 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-0[71518]: 2024-09-20T22:24:23.405+0000 7fb3bd739640 -1 osd.0 0 read_superblock omap replica is missing. 2024-09-20T22:24:23.843 INFO:teuthology.orchestra.run.smithi037.stdout: 5 171048 10.98 MB/sec execute 281 sec latency 198.902 ms 2024-09-20T22:24:24.607 INFO:journalctl@ceph.osd.0.smithi037.stdout:Sep 20 22:24:24 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-0[71518]: 2024-09-20T22:24:24.468+0000 7fb3bd739640 -1 osd.0 73 log_to_monitors true 2024-09-20T22:24:24.844 INFO:teuthology.orchestra.run.smithi037.stdout: 5 171252 10.97 MB/sec execute 282 sec latency 77.737 ms 2024-09-20T22:24:24.871 INFO:journalctl@ceph.osd.0.smithi037.stdout:Sep 20 22:24:24 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-0[71518]: 2024-09-20T22:24:24.720+0000 7fb3b54e4640 -1 osd.0 73 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-20T22:24:24.872 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:24 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:24:24.872 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:24 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:24:24.872 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:24 smithi037 ceph-mon[65640]: pgmap v66: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 325 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.3 MiB/s wr, 526 op/s; 208/1536 objects degraded (13.542%) 2024-09-20T22:24:24.872 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:24 smithi037 ceph-mon[65640]: Health check update: Degraded data redundancy: 216/1614 objects degraded (13.383%), 29 pgs degraded (PG_DEGRADED) 2024-09-20T22:24:24.872 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:24 smithi037 ceph-mon[65640]: from='osd.0 [v2:172.21.15.37:6802/1525927182,v1:172.21.15.37:6803/1525927182]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-20T22:24:24.872 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:24 smithi037 ceph-mon[65640]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-20T22:24:24.872 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:24 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:24:24.872 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:24 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:24:25.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:24 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:24:25.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:24 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:24:25.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:24 smithi194 ceph-mon[51725]: pgmap v66: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 325 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.3 MiB/s wr, 526 op/s; 208/1536 objects degraded (13.542%) 2024-09-20T22:24:25.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:24 smithi194 ceph-mon[51725]: Health check update: Degraded data redundancy: 216/1614 objects degraded (13.383%), 29 pgs degraded (PG_DEGRADED) 2024-09-20T22:24:25.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:24 smithi194 ceph-mon[51725]: from='osd.0 [v2:172.21.15.37:6802/1525927182,v1:172.21.15.37:6803/1525927182]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-20T22:24:25.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:24 smithi194 ceph-mon[51725]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-20T22:24:25.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:24 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:24:25.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:24 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:24:25.843 INFO:teuthology.orchestra.run.smithi037.stdout: 5 171522 10.95 MB/sec execute 283 sec latency 169.274 ms 2024-09-20T22:24:25.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:25 smithi037 ceph-mon[65640]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-20T22:24:25.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:25 smithi037 ceph-mon[65640]: from='osd.0 [v2:172.21.15.37:6802/1525927182,v1:172.21.15.37:6803/1525927182]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi037", "root=default"]}]: dispatch 2024-09-20T22:24:25.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:25 smithi037 ceph-mon[65640]: osdmap e76: 8 total, 7 up, 8 in 2024-09-20T22:24:25.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:25 smithi037 ceph-mon[65640]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi037", "root=default"]}]: dispatch 2024-09-20T22:24:26.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:25 smithi194 ceph-mon[51725]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-20T22:24:26.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:25 smithi194 ceph-mon[51725]: from='osd.0 [v2:172.21.15.37:6802/1525927182,v1:172.21.15.37:6803/1525927182]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi037", "root=default"]}]: dispatch 2024-09-20T22:24:26.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:25 smithi194 ceph-mon[51725]: osdmap e76: 8 total, 7 up, 8 in 2024-09-20T22:24:26.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:25 smithi194 ceph-mon[51725]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi037", "root=default"]}]: dispatch 2024-09-20T22:24:26.843 INFO:teuthology.orchestra.run.smithi037.stdout: 5 171562 10.91 MB/sec execute 284 sec latency 1078.567 ms 2024-09-20T22:24:26.925 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:26 smithi037 ceph-mon[65640]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-20T22:24:26.926 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:26 smithi037 ceph-mon[65640]: osd.0 [v2:172.21.15.37:6802/1525927182,v1:172.21.15.37:6803/1525927182] boot 2024-09-20T22:24:26.926 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:26 smithi037 ceph-mon[65640]: osdmap e77: 8 total, 8 up, 8 in 2024-09-20T22:24:26.926 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:26 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-20T22:24:26.926 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:26 smithi037 ceph-mon[65640]: pgmap v69: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 325 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.2 KiB/s rd, 3.4 MiB/s wr, 369 op/s; 208/1536 objects degraded (13.542%) 2024-09-20T22:24:27.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:26 smithi194 ceph-mon[51725]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-20T22:24:27.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:26 smithi194 ceph-mon[51725]: osd.0 [v2:172.21.15.37:6802/1525927182,v1:172.21.15.37:6803/1525927182] boot 2024-09-20T22:24:27.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:26 smithi194 ceph-mon[51725]: osdmap e77: 8 total, 8 up, 8 in 2024-09-20T22:24:27.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:26 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-20T22:24:27.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:26 smithi194 ceph-mon[51725]: pgmap v69: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 325 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.2 KiB/s rd, 3.4 MiB/s wr, 369 op/s; 208/1536 objects degraded (13.542%) 2024-09-20T22:24:27.694 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:27 smithi037 ceph-mon[65640]: osdmap e78: 8 total, 8 up, 8 in 2024-09-20T22:24:27.844 INFO:teuthology.orchestra.run.smithi037.stdout: 5 171762 10.88 MB/sec execute 285 sec latency 782.628 ms 2024-09-20T22:24:27.995 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:27 smithi194 ceph-mon[51725]: osdmap e78: 8 total, 8 up, 8 in 2024-09-20T22:24:28.720 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:28 smithi037 ceph-mon[65640]: osdmap e79: 8 total, 8 up, 8 in 2024-09-20T22:24:28.720 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:28 smithi037 ceph-mon[65640]: pgmap v72: 97 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+undersized, 14 active+undersized+degraded, 74 active+clean; 330 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 2.1 MiB/s wr, 263 op/s; 164/1557 objects degraded (10.533%); 2.1 MiB/s, 10 objects/s recovering 2024-09-20T22:24:28.844 INFO:teuthology.orchestra.run.smithi037.stdout: 5 172013 10.86 MB/sec execute 286 sec latency 40.766 ms 2024-09-20T22:24:29.007 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:28 smithi194 ceph-mon[51725]: osdmap e79: 8 total, 8 up, 8 in 2024-09-20T22:24:29.007 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:28 smithi194 ceph-mon[51725]: pgmap v72: 97 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+undersized, 14 active+undersized+degraded, 74 active+clean; 330 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 2.1 MiB/s wr, 263 op/s; 164/1557 objects degraded (10.533%); 2.1 MiB/s, 10 objects/s recovering 2024-09-20T22:24:29.753 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:29 smithi037 ceph-mon[65640]: osdmap e80: 8 total, 8 up, 8 in 2024-09-20T22:24:29.754 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:29 smithi037 ceph-mon[65640]: Health check update: Degraded data redundancy: 164/1557 objects degraded (10.533%), 17 pgs degraded (PG_DEGRADED) 2024-09-20T22:24:29.754 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:29 smithi037 ceph-mon[65640]: osdmap e81: 8 total, 8 up, 8 in 2024-09-20T22:24:29.844 INFO:teuthology.orchestra.run.smithi037.stdout: 5 172229 10.85 MB/sec execute 287 sec latency 130.451 ms 2024-09-20T22:24:30.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:29 smithi194 ceph-mon[51725]: osdmap e80: 8 total, 8 up, 8 in 2024-09-20T22:24:30.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:29 smithi194 ceph-mon[51725]: Health check update: Degraded data redundancy: 164/1557 objects degraded (10.533%), 17 pgs degraded (PG_DEGRADED) 2024-09-20T22:24:30.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:29 smithi194 ceph-mon[51725]: osdmap e81: 8 total, 8 up, 8 in 2024-09-20T22:24:30.776 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:30 smithi037 ceph-mon[65640]: pgmap v75: 97 pgs: 1 peering, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 89 active+clean; 329 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 2.9 MiB/s wr, 306 op/s; 92/1527 objects degraded (6.025%); 2.1 MiB/s, 34 objects/s recovering 2024-09-20T22:24:30.777 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:30 smithi037 ceph-mon[65640]: osdmap e82: 8 total, 8 up, 8 in 2024-09-20T22:24:30.844 INFO:teuthology.orchestra.run.smithi037.stdout: 5 172454 10.85 MB/sec execute 288 sec latency 787.796 ms 2024-09-20T22:24:31.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:30 smithi194 ceph-mon[51725]: pgmap v75: 97 pgs: 1 peering, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 89 active+clean; 329 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 2.9 MiB/s wr, 306 op/s; 92/1527 objects degraded (6.025%); 2.1 MiB/s, 34 objects/s recovering 2024-09-20T22:24:31.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:30 smithi194 ceph-mon[51725]: osdmap e82: 8 total, 8 up, 8 in 2024-09-20T22:24:31.844 INFO:teuthology.orchestra.run.smithi037.stdout: 5 172818 10.83 MB/sec execute 289 sec latency 930.234 ms 2024-09-20T22:24:32.544 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:32 smithi037 ceph-mon[65640]: osdmap e83: 8 total, 8 up, 8 in 2024-09-20T22:24:32.544 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:32 smithi037 ceph-mon[65640]: pgmap v78: 97 pgs: 1 peering, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 89 active+clean; 329 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 839 KiB/s wr, 43 op/s; 92/1527 objects degraded (6.025%); 0 B/s, 24 objects/s recovering 2024-09-20T22:24:32.544 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:32 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:24:32.544 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:32 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:24:32.544 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:32 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:24:32.544 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:32 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:24:32.544 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:32 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:24:32.544 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:32 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:24:32.844 INFO:teuthology.orchestra.run.smithi037.stdout: 5 173255 10.82 MB/sec execute 290 sec latency 60.458 ms 2024-09-20T22:24:32.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:32 smithi194 ceph-mon[51725]: osdmap e83: 8 total, 8 up, 8 in 2024-09-20T22:24:32.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:32 smithi194 ceph-mon[51725]: pgmap v78: 97 pgs: 1 peering, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 89 active+clean; 329 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 839 KiB/s wr, 43 op/s; 92/1527 objects degraded (6.025%); 0 B/s, 24 objects/s recovering 2024-09-20T22:24:32.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:32 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:24:32.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:32 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:24:32.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:32 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:24:32.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:32 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:24:32.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:32 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:24:32.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:32 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:24:33.564 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:33 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:24:33.565 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:33 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:24:33.565 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:33 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:24:33.565 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:33 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:24:33.565 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:33 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-20T22:24:33.565 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:33 smithi037 ceph-mon[65640]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-20T22:24:33.565 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:33 smithi037 ceph-mon[65640]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-09-20T22:24:33.844 INFO:teuthology.orchestra.run.smithi037.stdout: 5 173538 10.81 MB/sec execute 291 sec latency 54.699 ms 2024-09-20T22:24:33.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:33 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:24:33.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:33 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:24:33.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:33 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:24:33.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:33 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:24:33.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:33 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-20T22:24:33.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:33 smithi194 ceph-mon[51725]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-20T22:24:33.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:33 smithi194 ceph-mon[51725]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-09-20T22:24:34.575 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:34 smithi037 ceph-mon[65640]: pgmap v79: 97 pgs: 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 325 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.5 MiB/s wr, 422 op/s; 21/1509 objects degraded (1.392%); 20 KiB/s, 31 objects/s recovering 2024-09-20T22:24:34.575 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:34 smithi037 ceph-mon[65640]: Health check update: Degraded data redundancy: 92/1527 objects degraded (6.025%), 6 pgs degraded (PG_DEGRADED) 2024-09-20T22:24:34.844 INFO:teuthology.orchestra.run.smithi037.stdout: 5 174192 10.85 MB/sec execute 292 sec latency 24.184 ms 2024-09-20T22:24:34.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:34 smithi194 ceph-mon[51725]: pgmap v79: 97 pgs: 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 325 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.5 MiB/s wr, 422 op/s; 21/1509 objects degraded (1.392%); 20 KiB/s, 31 objects/s recovering 2024-09-20T22:24:34.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:34 smithi194 ceph-mon[51725]: Health check update: Degraded data redundancy: 92/1527 objects degraded (6.025%), 6 pgs degraded (PG_DEGRADED) 2024-09-20T22:24:35.844 INFO:teuthology.orchestra.run.smithi037.stdout: 5 174715 10.89 MB/sec execute 293 sec latency 36.758 ms 2024-09-20T22:24:36.089 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:35 smithi037 ceph-mon[65640]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 21/1509 objects degraded (1.392%), 1 pg degraded) 2024-09-20T22:24:36.089 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:35 smithi037 ceph-mon[65640]: Cluster is now healthy 2024-09-20T22:24:36.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:35 smithi194 ceph-mon[51725]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 21/1509 objects degraded (1.392%), 1 pg degraded) 2024-09-20T22:24:36.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:35 smithi194 ceph-mon[51725]: Cluster is now healthy 2024-09-20T22:24:36.844 INFO:teuthology.orchestra.run.smithi037.stdout: 5 175147 10.89 MB/sec execute 294 sec latency 32.076 ms 2024-09-20T22:24:37.093 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:36 smithi037 ceph-mon[65640]: pgmap v80: 97 pgs: 97 active+clean; 326 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.8 KiB/s rd, 3.5 MiB/s wr, 369 op/s; 17 KiB/s, 28 objects/s recovering 2024-09-20T22:24:37.093 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:36 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:24:37.093 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:36 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:24:37.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:36 smithi194 ceph-mon[51725]: pgmap v80: 97 pgs: 97 active+clean; 326 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.8 KiB/s rd, 3.5 MiB/s wr, 369 op/s; 17 KiB/s, 28 objects/s recovering 2024-09-20T22:24:37.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:36 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:24:37.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:36 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:24:37.844 INFO:teuthology.orchestra.run.smithi037.stdout: 5 175538 10.86 MB/sec execute 295 sec latency 44.273 ms 2024-09-20T22:24:38.844 INFO:teuthology.orchestra.run.smithi037.stdout: 5 175756 10.84 MB/sec execute 296 sec latency 87.360 ms 2024-09-20T22:24:39.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:38 smithi037 ceph-mon[65640]: pgmap v81: 97 pgs: 97 active+clean; 335 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 6.1 MiB/s wr, 657 op/s; 13 KiB/s, 11 objects/s recovering 2024-09-20T22:24:39.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:38 smithi194 ceph-mon[51725]: pgmap v81: 97 pgs: 97 active+clean; 335 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 6.1 MiB/s wr, 657 op/s; 13 KiB/s, 11 objects/s recovering 2024-09-20T22:24:39.844 INFO:teuthology.orchestra.run.smithi037.stdout: 5 176280 10.86 MB/sec execute 297 sec latency 33.094 ms 2024-09-20T22:24:40.844 INFO:teuthology.orchestra.run.smithi037.stdout: 5 176865 10.86 MB/sec execute 298 sec latency 33.430 ms 2024-09-20T22:24:41.181 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:40 smithi037 ceph-mon[65640]: pgmap v82: 97 pgs: 97 active+clean; 334 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 6.1 MiB/s wr, 596 op/s; 11 KiB/s, 9 objects/s recovering 2024-09-20T22:24:41.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:40 smithi194 ceph-mon[51725]: pgmap v82: 97 pgs: 97 active+clean; 334 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 6.1 MiB/s wr, 596 op/s; 11 KiB/s, 9 objects/s recovering 2024-09-20T22:24:41.844 INFO:teuthology.orchestra.run.smithi037.stdout: 5 177224 10.85 MB/sec execute 299 sec latency 44.202 ms 2024-09-20T22:24:42.844 INFO:teuthology.orchestra.run.smithi037.stdout: 5 177634 10.87 MB/sec execute 300 sec latency 48.891 ms 2024-09-20T22:24:42.961 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:42 smithi037 ceph-mon[65640]: pgmap v83: 97 pgs: 97 active+clean; 334 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.5 MiB/s wr, 538 op/s; 10 KiB/s, 8 objects/s recovering 2024-09-20T22:24:42.962 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:42 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:24:43.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:42 smithi194 ceph-mon[51725]: pgmap v83: 97 pgs: 97 active+clean; 334 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.5 MiB/s wr, 538 op/s; 10 KiB/s, 8 objects/s recovering 2024-09-20T22:24:43.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:42 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:24:43.844 INFO:teuthology.orchestra.run.smithi037.stdout: 5 178064 10.89 MB/sec execute 301 sec latency 56.547 ms 2024-09-20T22:24:44.844 INFO:teuthology.orchestra.run.smithi037.stdout: 5 178413 10.89 MB/sec execute 302 sec latency 37.779 ms 2024-09-20T22:24:44.996 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:44 smithi037 ceph-mon[65640]: pgmap v84: 97 pgs: 97 active+clean; 333 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 7.0 MiB/s wr, 709 op/s; 8.8 KiB/s, 7 objects/s recovering 2024-09-20T22:24:45.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:44 smithi194 ceph-mon[51725]: pgmap v84: 97 pgs: 97 active+clean; 333 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 7.0 MiB/s wr, 709 op/s; 8.8 KiB/s, 7 objects/s recovering 2024-09-20T22:24:45.844 INFO:teuthology.orchestra.run.smithi037.stdout: 5 178867 10.88 MB/sec execute 303 sec latency 71.276 ms 2024-09-20T22:24:46.844 INFO:teuthology.orchestra.run.smithi037.stdout: 5 179103 10.86 MB/sec execute 304 sec latency 43.016 ms 2024-09-20T22:24:46.946 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:46 smithi037 ceph-mon[65640]: pgmap v85: 97 pgs: 97 active+clean; 332 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 6.0 MiB/s wr, 544 op/s; 0 B/s, 1 objects/s recovering 2024-09-20T22:24:47.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:46 smithi194 ceph-mon[51725]: pgmap v85: 97 pgs: 97 active+clean; 332 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 6.0 MiB/s wr, 544 op/s; 0 B/s, 1 objects/s recovering 2024-09-20T22:24:47.845 INFO:teuthology.orchestra.run.smithi037.stdout: 5 179390 10.84 MB/sec execute 305 sec latency 45.904 ms 2024-09-20T22:24:47.973 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:47 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-20T22:24:48.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:47 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-20T22:24:48.410 INFO:teuthology.orchestra.run.smithi037.stdout:true 2024-09-20T22:24:48.845 INFO:teuthology.orchestra.run.smithi037.stdout: 5 179766 10.85 MB/sec execute 306 sec latency 59.122 ms 2024-09-20T22:24:49.013 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:48 smithi037 ceph-mon[65640]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-20T22:24:49.013 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:48 smithi037 ceph-mon[65640]: Upgrade: osd.1 is safe to restart 2024-09-20T22:24:49.014 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:48 smithi037 ceph-mon[65640]: pgmap v86: 97 pgs: 97 active+clean; 331 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.4 MiB/s wr, 713 op/s 2024-09-20T22:24:49.344 INFO:teuthology.orchestra.run.smithi037.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T22:24:49.344 INFO:teuthology.orchestra.run.smithi037.stdout:alertmanager.smithi037 smithi037 *:9093,9094 running (4m) 25s ago 13m 25.8M - 0.25.0 c8568f914cd2 d1f99b3cd707 2024-09-20T22:24:49.344 INFO:teuthology.orchestra.run.smithi037.stdout:crash.smithi037 smithi037 running (78s) 25s ago 13m 6895k - 19.1.1-367-g9e2ac70b cc9251b744a3 77e510d8bdb1 2024-09-20T22:24:49.345 INFO:teuthology.orchestra.run.smithi037.stdout:crash.smithi194 smithi194 running (71s) 67s ago 13m 6895k - 19.1.1-367-g9e2ac70b cc9251b744a3 4d169d1309bc 2024-09-20T22:24:49.345 INFO:teuthology.orchestra.run.smithi037.stdout:grafana.smithi037 smithi037 *:3000 running (4m) 25s ago 13m 54.7M - dad864ee21e9 2306f3c2da19 2024-09-20T22:24:49.345 INFO:teuthology.orchestra.run.smithi037.stdout:mds.foofs.smithi037.sdtedi smithi037 running (7m) 25s ago 7m 38.0M - 17.2.0 e1d6a67b021e 0a7898d25b8d 2024-09-20T22:24:49.345 INFO:teuthology.orchestra.run.smithi037.stdout:mds.foofs.smithi194.lkroso smithi194 running (7m) 67s ago 7m 15.6M - 17.2.0 e1d6a67b021e 17736de30008 2024-09-20T22:24:49.345 INFO:teuthology.orchestra.run.smithi037.stdout:mgr.smithi037.lykblh smithi037 *:8443,9283,8765 running (3m) 25s ago 14m 500M - 19.1.1-367-g9e2ac70b cc9251b744a3 181e46fd9ba6 2024-09-20T22:24:49.345 INFO:teuthology.orchestra.run.smithi037.stdout:mgr.smithi194.bkcglt smithi194 *:8443,9283,8765 running (2m) 67s ago 12m 439M - 19.1.1-367-g9e2ac70b cc9251b744a3 1fa610c97517 2024-09-20T22:24:49.345 INFO:teuthology.orchestra.run.smithi037.stdout:mon.smithi037 smithi037 running (2m) 25s ago 15m 46.7M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 c0229eceb527 2024-09-20T22:24:49.345 INFO:teuthology.orchestra.run.smithi037.stdout:mon.smithi194 smithi194 running (2m) 67s ago 12m 37.2M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 97854807ba91 2024-09-20T22:24:49.345 INFO:teuthology.orchestra.run.smithi037.stdout:nfs.foo.0.0.smithi037.spabdc smithi037 *:2049 running (7m) 25s ago 7m 266M - 4.0 e1d6a67b021e 858ff9d558a2 2024-09-20T22:24:49.345 INFO:teuthology.orchestra.run.smithi037.stdout:nfs.foo.1.0.smithi194.qzjyej smithi194 *:2049 running (7m) 67s ago 7m 89.8M - 4.0 e1d6a67b021e c07fe0eb7c75 2024-09-20T22:24:49.345 INFO:teuthology.orchestra.run.smithi037.stdout:node-exporter.smithi037 smithi037 *:9100 running (4m) 25s ago 13m 16.0M - 1.5.0 0da6a335fe13 433e6aaf6137 2024-09-20T22:24:49.345 INFO:teuthology.orchestra.run.smithi037.stdout:node-exporter.smithi194 smithi194 *:9100 running (4m) 67s ago 12m 15.7M - 1.5.0 0da6a335fe13 1d30bb810ad1 2024-09-20T22:24:49.345 INFO:teuthology.orchestra.run.smithi037.stdout:osd.0 smithi037 running (35s) 25s ago 12m 11.3M 1937M 19.1.1-367-g9e2ac70b cc9251b744a3 c5568bd43c07 2024-09-20T22:24:49.346 INFO:teuthology.orchestra.run.smithi037.stdout:osd.1 smithi037 running (11m) 25s ago 11m 521M 1937M 17.2.0 e1d6a67b021e 09647c133032 2024-09-20T22:24:49.346 INFO:teuthology.orchestra.run.smithi037.stdout:osd.2 smithi037 running (11m) 25s ago 11m 397M 1937M 17.2.0 e1d6a67b021e 6c15b1175256 2024-09-20T22:24:49.346 INFO:teuthology.orchestra.run.smithi037.stdout:osd.3 smithi037 running (10m) 25s ago 10m 522M 1937M 17.2.0 e1d6a67b021e 3a22952c6475 2024-09-20T22:24:49.346 INFO:teuthology.orchestra.run.smithi037.stdout:osd.4 smithi194 running (10m) 67s ago 10m 513M 2704M 17.2.0 e1d6a67b021e fa107be8915c 2024-09-20T22:24:49.346 INFO:teuthology.orchestra.run.smithi037.stdout:osd.5 smithi194 running (9m) 67s ago 9m 324M 2704M 17.2.0 e1d6a67b021e 290b21cbd131 2024-09-20T22:24:49.346 INFO:teuthology.orchestra.run.smithi037.stdout:osd.6 smithi194 running (9m) 67s ago 9m 511M 2704M 17.2.0 e1d6a67b021e 27f0720639ed 2024-09-20T22:24:49.346 INFO:teuthology.orchestra.run.smithi037.stdout:osd.7 smithi194 running (9m) 67s ago 9m 406M 2704M 17.2.0 e1d6a67b021e 62d163f4fd92 2024-09-20T22:24:49.346 INFO:teuthology.orchestra.run.smithi037.stdout:prometheus.smithi037 smithi037 *:9095 running (3m) 25s ago 12m 63.7M - 2.43.0 a07b618ecd1d 05b826dde2ff 2024-09-20T22:24:49.346 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:48 smithi194 ceph-mon[51725]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-20T22:24:49.346 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:48 smithi194 ceph-mon[51725]: Upgrade: osd.1 is safe to restart 2024-09-20T22:24:49.346 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:48 smithi194 ceph-mon[51725]: pgmap v86: 97 pgs: 97 active+clean; 331 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.4 MiB/s wr, 713 op/s 2024-09-20T22:24:49.844 INFO:teuthology.orchestra.run.smithi037.stdout: 5 180136 10.83 MB/sec execute 307 sec latency 69.692 ms 2024-09-20T22:24:49.955 INFO:teuthology.orchestra.run.smithi037.stdout:{ 2024-09-20T22:24:49.955 INFO:teuthology.orchestra.run.smithi037.stdout: "mon": { 2024-09-20T22:24:49.955 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 2 2024-09-20T22:24:49.955 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:24:49.955 INFO:teuthology.orchestra.run.smithi037.stdout: "mgr": { 2024-09-20T22:24:49.955 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 2 2024-09-20T22:24:49.955 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:24:49.955 INFO:teuthology.orchestra.run.smithi037.stdout: "osd": { 2024-09-20T22:24:49.955 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-09-20T22:24:49.956 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 1 2024-09-20T22:24:49.956 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:24:49.956 INFO:teuthology.orchestra.run.smithi037.stdout: "mds": { 2024-09-20T22:24:49.956 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T22:24:49.956 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:24:49.956 INFO:teuthology.orchestra.run.smithi037.stdout: "overall": { 2024-09-20T22:24:49.956 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 9, 2024-09-20T22:24:49.956 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 5 2024-09-20T22:24:49.956 INFO:teuthology.orchestra.run.smithi037.stdout: } 2024-09-20T22:24:49.956 INFO:teuthology.orchestra.run.smithi037.stdout:} 2024-09-20T22:24:50.028 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:49 smithi037 ceph-mon[65640]: from='client.34222 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:24:50.029 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:49 smithi037 ceph-mon[65640]: from='client.34226 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:24:50.029 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:49 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:24:50.029 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:49 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-20T22:24:50.029 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:49 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:24:50.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:49 smithi194 ceph-mon[51725]: from='client.34222 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:24:50.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:49 smithi194 ceph-mon[51725]: from='client.34226 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:24:50.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:49 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:24:50.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:49 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-20T22:24:50.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:49 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:24:50.560 INFO:teuthology.orchestra.run.smithi037.stdout:{ 2024-09-20T22:24:50.560 INFO:teuthology.orchestra.run.smithi037.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5", 2024-09-20T22:24:50.561 INFO:teuthology.orchestra.run.smithi037.stdout: "in_progress": true, 2024-09-20T22:24:50.561 INFO:teuthology.orchestra.run.smithi037.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-20T22:24:50.561 INFO:teuthology.orchestra.run.smithi037.stdout: "services_complete": [ 2024-09-20T22:24:50.561 INFO:teuthology.orchestra.run.smithi037.stdout: "mon", 2024-09-20T22:24:50.561 INFO:teuthology.orchestra.run.smithi037.stdout: "mgr", 2024-09-20T22:24:50.561 INFO:teuthology.orchestra.run.smithi037.stdout: "crash" 2024-09-20T22:24:50.561 INFO:teuthology.orchestra.run.smithi037.stdout: ], 2024-09-20T22:24:50.561 INFO:teuthology.orchestra.run.smithi037.stdout: "progress": "7/23 daemons upgraded", 2024-09-20T22:24:50.561 INFO:teuthology.orchestra.run.smithi037.stdout: "message": "Currently upgrading osd daemons", 2024-09-20T22:24:50.561 INFO:teuthology.orchestra.run.smithi037.stdout: "is_paused": false 2024-09-20T22:24:50.561 INFO:teuthology.orchestra.run.smithi037.stdout:} 2024-09-20T22:24:50.844 INFO:teuthology.orchestra.run.smithi037.stdout: 5 180576 10.83 MB/sec execute 308 sec latency 51.213 ms 2024-09-20T22:24:51.051 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:50 smithi037 ceph-mon[65640]: from='client.34230 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:24:51.051 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:50 smithi037 ceph-mon[65640]: Upgrade: Updating osd.1 2024-09-20T22:24:51.051 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:50 smithi037 ceph-mon[65640]: Deploying daemon osd.1 on smithi037 2024-09-20T22:24:51.051 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:50 smithi037 ceph-mon[65640]: pgmap v87: 97 pgs: 97 active+clean; 330 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.6 MiB/s wr, 509 op/s 2024-09-20T22:24:51.051 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:50 smithi037 ceph-mon[65640]: from='client.? 172.21.15.37:0/207080859' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:24:51.204 INFO:teuthology.orchestra.run.smithi037.stdout:HEALTH_OK 2024-09-20T22:24:51.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:50 smithi194 ceph-mon[51725]: from='client.34230 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:24:51.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:50 smithi194 ceph-mon[51725]: Upgrade: Updating osd.1 2024-09-20T22:24:51.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:50 smithi194 ceph-mon[51725]: Deploying daemon osd.1 on smithi037 2024-09-20T22:24:51.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:50 smithi194 ceph-mon[51725]: pgmap v87: 97 pgs: 97 active+clean; 330 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.6 MiB/s wr, 509 op/s 2024-09-20T22:24:51.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:50 smithi194 ceph-mon[51725]: from='client.? 172.21.15.37:0/207080859' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:24:51.854 INFO:teuthology.orchestra.run.smithi037.stdout: 5 180797 10.81 MB/sec execute 309 sec latency 39.353 ms 2024-09-20T22:24:52.076 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:51 smithi037 ceph-mon[65640]: from='client.44175 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:24:52.077 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:51 smithi037 ceph-mon[65640]: from='client.? 172.21.15.37:0/3634357756' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T22:24:52.077 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:51 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:24:52.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:51 smithi194 ceph-mon[51725]: from='client.44175 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:24:52.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:51 smithi194 ceph-mon[51725]: from='client.? 172.21.15.37:0/3634357756' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T22:24:52.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:51 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:24:52.854 INFO:teuthology.orchestra.run.smithi037.stdout: 5 181105 10.82 MB/sec execute 310 sec latency 63.304 ms 2024-09-20T22:24:53.102 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:52 smithi037 ceph-mon[65640]: pgmap v88: 97 pgs: 97 active+clean; 330 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 4.9 MiB/s wr, 482 op/s 2024-09-20T22:24:53.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:52 smithi194 ceph-mon[51725]: pgmap v88: 97 pgs: 97 active+clean; 330 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 4.9 MiB/s wr, 482 op/s 2024-09-20T22:24:53.854 INFO:teuthology.orchestra.run.smithi037.stdout: 5 181375 10.81 MB/sec execute 311 sec latency 82.666 ms 2024-09-20T22:24:54.854 INFO:teuthology.orchestra.run.smithi037.stdout: 5 181663 10.81 MB/sec execute 312 sec latency 65.632 ms 2024-09-20T22:24:55.187 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:54 smithi037 ceph-mon[65640]: pgmap v89: 97 pgs: 97 active+clean; 330 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.5 MiB/s wr, 665 op/s 2024-09-20T22:24:55.187 INFO:journalctl@ceph.osd.1.smithi037.stdout:Sep 20 22:24:55 smithi037 systemd[1]: Stopping Ceph osd.1 for f9a6357a-779c-11ef-baf5-efdc52797490... 2024-09-20T22:24:55.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:54 smithi194 ceph-mon[51725]: pgmap v89: 97 pgs: 97 active+clean; 330 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.5 MiB/s wr, 665 op/s 2024-09-20T22:24:55.692 INFO:journalctl@ceph.osd.1.smithi037.stdout:Sep 20 22:24:55 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-1[37075]: 2024-09-20T22:24:55.546+0000 7f701a386700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-20T22:24:55.692 INFO:journalctl@ceph.osd.1.smithi037.stdout:Sep 20 22:24:55 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-1[37075]: 2024-09-20T22:24:55.546+0000 7f701a386700 -1 osd.1 83 *** Got signal Terminated *** 2024-09-20T22:24:55.692 INFO:journalctl@ceph.osd.1.smithi037.stdout:Sep 20 22:24:55 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-1[37075]: 2024-09-20T22:24:55.546+0000 7f701a386700 -1 osd.1 83 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-20T22:24:55.854 INFO:teuthology.orchestra.run.smithi037.stdout: 5 182109 10.83 MB/sec execute 313 sec latency 39.375 ms 2024-09-20T22:24:56.193 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:55 smithi037 ceph-mon[65640]: osd.1 marked itself down and dead 2024-09-20T22:24:56.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:55 smithi194 ceph-mon[51725]: osd.1 marked itself down and dead 2024-09-20T22:24:56.762 INFO:journalctl@ceph.osd.1.smithi037.stdout:Sep 20 22:24:56 smithi037 podman[73784]: 2024-09-20 22:24:56.59180921 +0000 UTC m=+1.309048206 container died 09647c133032a22f93ae2d7cb61e82c18fe3914bbd91e921c2ff289848eadbbf (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-1, com.redhat.component=centos-stream-container, ceph=True, GIT_BRANCH=HEAD, version=8, distribution-scope=public, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, RELEASE=HEAD, io.buildah.version=1.19.8) 2024-09-20T22:24:56.854 INFO:teuthology.orchestra.run.smithi037.stdout: 5 182330 10.80 MB/sec execute 314 sec latency 875.110 ms 2024-09-20T22:24:57.012 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:56 smithi037 ceph-mon[65640]: pgmap v90: 97 pgs: 97 active+clean; 330 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.5 MiB/s wr, 432 op/s 2024-09-20T22:24:57.013 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:56 smithi037 ceph-mon[65640]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-20T22:24:57.013 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:56 smithi037 ceph-mon[65640]: osdmap e84: 8 total, 7 up, 8 in 2024-09-20T22:24:57.246 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:56 smithi194 ceph-mon[51725]: pgmap v90: 97 pgs: 97 active+clean; 330 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.5 MiB/s wr, 432 op/s 2024-09-20T22:24:57.246 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:56 smithi194 ceph-mon[51725]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-20T22:24:57.246 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:56 smithi194 ceph-mon[51725]: osdmap e84: 8 total, 7 up, 8 in 2024-09-20T22:24:57.264 INFO:journalctl@ceph.osd.1.smithi037.stdout:Sep 20 22:24:57 smithi037 podman[73784]: 2024-09-20 22:24:57.177576412 +0000 UTC m=+1.894815405 container cleanup 09647c133032a22f93ae2d7cb61e82c18fe3914bbd91e921c2ff289848eadbbf (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-1, build-date=2022-05-03T08:36:31.336870, release=754, architecture=x86_64, vendor=Red Hat, Inc., name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.openshift.expose-services=, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, version=8, ceph=True, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD) 2024-09-20T22:24:57.264 INFO:journalctl@ceph.osd.1.smithi037.stdout:Sep 20 22:24:57 smithi037 bash[73784]: ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-1 2024-09-20T22:24:57.516 INFO:journalctl@ceph.osd.1.smithi037.stdout:Sep 20 22:24:57 smithi037 podman[73800]: 2024-09-20 22:24:57.441204418 +0000 UTC m=+0.846370662 container remove 09647c133032a22f93ae2d7cb61e82c18fe3914bbd91e921c2ff289848eadbbf (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-1, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, distribution-scope=public, maintainer=Guillaume Abrioux , name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., vcs-type=git, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, ceph=True, GIT_BRANCH=HEAD) 2024-09-20T22:24:57.854 INFO:teuthology.orchestra.run.smithi037.stdout: 5 182581 10.78 MB/sec execute 315 sec latency 1006.502 ms 2024-09-20T22:24:58.025 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:57 smithi037 ceph-mon[65640]: osdmap e85: 8 total, 7 up, 8 in 2024-09-20T22:24:58.280 INFO:journalctl@ceph.osd.1.smithi037.stdout:Sep 20 22:24:58 smithi037 podman[73879]: 2024-09-20 22:24:57.969515623 +0000 UTC m=+0.027989974 image pull cc9251b744a3faafca7c3095f79bfff81873afe823f31a03a11e015a34abcdbd quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5 2024-09-20T22:24:58.280 INFO:journalctl@ceph.osd.1.smithi037.stdout:Sep 20 22:24:58 smithi037 podman[73879]: 2024-09-20 22:24:58.141892809 +0000 UTC m=+0.200367143 container create a2b23814b84d8a0d4dcecc34bac79ab26f026cf74597de3b8821139755938546 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-1-deactivate, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, RELEASE=squid-9e2ac70, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-09-20T22:24:58.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:57 smithi194 ceph-mon[51725]: osdmap e85: 8 total, 7 up, 8 in 2024-09-20T22:24:58.787 INFO:journalctl@ceph.osd.1.smithi037.stdout:Sep 20 22:24:58 smithi037 podman[73879]: 2024-09-20 22:24:58.589465685 +0000 UTC m=+0.647940052 container init a2b23814b84d8a0d4dcecc34bac79ab26f026cf74597de3b8821139755938546 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-1-deactivate, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, RELEASE=squid-9e2ac70, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-09-20T22:24:58.787 INFO:journalctl@ceph.osd.1.smithi037.stdout:Sep 20 22:24:58 smithi037 podman[73879]: 2024-09-20 22:24:58.598540237 +0000 UTC m=+0.657014574 container start a2b23814b84d8a0d4dcecc34bac79ab26f026cf74597de3b8821139755938546 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-1-deactivate, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=squid-9e2ac70, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2) 2024-09-20T22:24:58.787 INFO:journalctl@ceph.osd.1.smithi037.stdout:Sep 20 22:24:58 smithi037 podman[73879]: 2024-09-20 22:24:58.776996966 +0000 UTC m=+0.835471303 container attach a2b23814b84d8a0d4dcecc34bac79ab26f026cf74597de3b8821139755938546 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-1-deactivate, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, RELEASE=squid-9e2ac70, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-09-20T22:24:58.854 INFO:teuthology.orchestra.run.smithi037.stdout: 5 182907 10.77 MB/sec execute 316 sec latency 35.597 ms 2024-09-20T22:24:59.042 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:58 smithi037 ceph-mon[65640]: pgmap v93: 97 pgs: 13 active+undersized, 4 stale+active+clean, 17 active+undersized+degraded, 63 active+clean; 333 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.2 MiB/s wr, 542 op/s; 131/1593 objects degraded (8.223%) 2024-09-20T22:24:59.042 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:24:58 smithi037 ceph-mon[65640]: Health check failed: Degraded data redundancy: 131/1593 objects degraded (8.223%), 17 pgs degraded (PG_DEGRADED) 2024-09-20T22:24:59.042 INFO:journalctl@ceph.osd.1.smithi037.stdout:Sep 20 22:24:58 smithi037 podman[73879]: 2024-09-20 22:24:58.994048549 +0000 UTC m=+1.052522897 container died a2b23814b84d8a0d4dcecc34bac79ab26f026cf74597de3b8821139755938546 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-1-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , RELEASE=squid-9e2ac70, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-20T22:24:59.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:58 smithi194 ceph-mon[51725]: pgmap v93: 97 pgs: 13 active+undersized, 4 stale+active+clean, 17 active+undersized+degraded, 63 active+clean; 333 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.2 MiB/s wr, 542 op/s; 131/1593 objects degraded (8.223%) 2024-09-20T22:24:59.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:24:58 smithi194 ceph-mon[51725]: Health check failed: Degraded data redundancy: 131/1593 objects degraded (8.223%), 17 pgs degraded (PG_DEGRADED) 2024-09-20T22:24:59.854 INFO:teuthology.orchestra.run.smithi037.stdout: 5 183232 10.75 MB/sec execute 317 sec latency 68.096 ms 2024-09-20T22:25:00.332 INFO:journalctl@ceph.osd.1.smithi037.stdout:Sep 20 22:25:00 smithi037 podman[73900]: 2024-09-20 22:25:00.182726206 +0000 UTC m=+1.179687822 container remove a2b23814b84d8a0d4dcecc34bac79ab26f026cf74597de3b8821139755938546 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-1-deactivate, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, RELEASE=squid-9e2ac70, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-20T22:25:00.332 INFO:journalctl@ceph.osd.1.smithi037.stdout:Sep 20 22:25:00 smithi037 systemd[1]: ceph-f9a6357a-779c-11ef-baf5-efdc52797490@osd.1.service: Deactivated successfully. 2024-09-20T22:25:00.332 INFO:journalctl@ceph.osd.1.smithi037.stdout:Sep 20 22:25:00 smithi037 systemd[1]: Stopped Ceph osd.1 for f9a6357a-779c-11ef-baf5-efdc52797490. 2024-09-20T22:25:00.332 INFO:journalctl@ceph.osd.1.smithi037.stdout:Sep 20 22:25:00 smithi037 systemd[1]: ceph-f9a6357a-779c-11ef-baf5-efdc52797490@osd.1.service: Consumed 3min 23.073s CPU time. 2024-09-20T22:25:00.838 INFO:journalctl@ceph.osd.1.smithi037.stdout:Sep 20 22:25:00 smithi037 systemd[1]: Starting Ceph osd.1 for f9a6357a-779c-11ef-baf5-efdc52797490... 2024-09-20T22:25:00.854 INFO:teuthology.orchestra.run.smithi037.stdout: 5 183562 10.75 MB/sec execute 318 sec latency 58.608 ms 2024-09-20T22:25:01.093 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:00 smithi037 ceph-mon[65640]: pgmap v94: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 333 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.8 MiB/s wr, 570 op/s; 174/1596 objects degraded (10.902%) 2024-09-20T22:25:01.093 INFO:journalctl@ceph.osd.1.smithi037.stdout:Sep 20 22:25:00 smithi037 podman[73992]: 2024-09-20 22:25:00.797408388 +0000 UTC m=+0.029631985 image pull cc9251b744a3faafca7c3095f79bfff81873afe823f31a03a11e015a34abcdbd quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5 2024-09-20T22:25:01.093 INFO:journalctl@ceph.osd.1.smithi037.stdout:Sep 20 22:25:01 smithi037 podman[73992]: 2024-09-20 22:25:01.043617542 +0000 UTC m=+0.275841116 container create 60bfeb71df3b549f5928890e3b6a455fc1d54003d3c45253f7c6efb630d741a7 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-1-activate, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, RELEASE=squid-9e2ac70, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS) 2024-09-20T22:25:01.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:00 smithi194 ceph-mon[51725]: pgmap v94: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 333 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.8 MiB/s wr, 570 op/s; 174/1596 objects degraded (10.902%) 2024-09-20T22:25:01.595 INFO:journalctl@ceph.osd.1.smithi037.stdout:Sep 20 22:25:01 smithi037 podman[73992]: 2024-09-20 22:25:01.449866569 +0000 UTC m=+0.682090145 container init 60bfeb71df3b549f5928890e3b6a455fc1d54003d3c45253f7c6efb630d741a7 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-1-activate, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-9e2ac70, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-09-20T22:25:01.595 INFO:journalctl@ceph.osd.1.smithi037.stdout:Sep 20 22:25:01 smithi037 podman[73992]: 2024-09-20 22:25:01.463106391 +0000 UTC m=+0.695329963 container start 60bfeb71df3b549f5928890e3b6a455fc1d54003d3c45253f7c6efb630d741a7 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-1-activate, RELEASE=squid-9e2ac70, org.label-schema.vendor=CentOS, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240731) 2024-09-20T22:25:01.852 INFO:journalctl@ceph.osd.1.smithi037.stdout:Sep 20 22:25:01 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-1-activate[74004]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T22:25:01.852 INFO:journalctl@ceph.osd.1.smithi037.stdout:Sep 20 22:25:01 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-1-activate[74004]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T22:25:01.852 INFO:journalctl@ceph.osd.1.smithi037.stdout:Sep 20 22:25:01 smithi037 bash[73992]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T22:25:01.852 INFO:journalctl@ceph.osd.1.smithi037.stdout:Sep 20 22:25:01 smithi037 bash[73992]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T22:25:01.852 INFO:journalctl@ceph.osd.1.smithi037.stdout:Sep 20 22:25:01 smithi037 podman[73992]: 2024-09-20 22:25:01.739299963 +0000 UTC m=+0.971523550 container attach 60bfeb71df3b549f5928890e3b6a455fc1d54003d3c45253f7c6efb630d741a7 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-1-activate, RELEASE=squid-9e2ac70, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-20T22:25:01.854 INFO:teuthology.orchestra.run.smithi037.stdout: 5 184094 10.77 MB/sec execute 319 sec latency 72.508 ms 2024-09-20T22:25:02.854 INFO:teuthology.orchestra.run.smithi037.stdout: 5 184485 10.76 MB/sec execute 320 sec latency 78.825 ms 2024-09-20T22:25:03.148 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:02 smithi037 ceph-mon[65640]: pgmap v95: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 333 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 3.4 MiB/s wr, 294 op/s; 174/1596 objects degraded (10.902%) 2024-09-20T22:25:03.149 INFO:journalctl@ceph.osd.1.smithi037.stdout:Sep 20 22:25:02 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-1-activate[74004]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-20T22:25:03.149 INFO:journalctl@ceph.osd.1.smithi037.stdout:Sep 20 22:25:02 smithi037 bash[73992]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-20T22:25:03.149 INFO:journalctl@ceph.osd.1.smithi037.stdout:Sep 20 22:25:02 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-1-activate[74004]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-20T22:25:03.149 INFO:journalctl@ceph.osd.1.smithi037.stdout:Sep 20 22:25:02 smithi037 bash[73992]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-20T22:25:03.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:02 smithi194 ceph-mon[51725]: pgmap v95: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 333 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 3.4 MiB/s wr, 294 op/s; 174/1596 objects degraded (10.902%) 2024-09-20T22:25:03.401 INFO:journalctl@ceph.osd.1.smithi037.stdout:Sep 20 22:25:03 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-1-activate[74004]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-20T22:25:03.401 INFO:journalctl@ceph.osd.1.smithi037.stdout:Sep 20 22:25:03 smithi037 bash[73992]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-20T22:25:03.401 INFO:journalctl@ceph.osd.1.smithi037.stdout:Sep 20 22:25:03 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-1-activate[74004]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-20T22:25:03.402 INFO:journalctl@ceph.osd.1.smithi037.stdout:Sep 20 22:25:03 smithi037 bash[73992]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-20T22:25:03.402 INFO:journalctl@ceph.osd.1.smithi037.stdout:Sep 20 22:25:03 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-1-activate[74004]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-09-20T22:25:03.402 INFO:journalctl@ceph.osd.1.smithi037.stdout:Sep 20 22:25:03 smithi037 bash[73992]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-09-20T22:25:03.402 INFO:journalctl@ceph.osd.1.smithi037.stdout:Sep 20 22:25:03 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-1-activate[74004]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-20T22:25:03.402 INFO:journalctl@ceph.osd.1.smithi037.stdout:Sep 20 22:25:03 smithi037 bash[73992]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-20T22:25:03.402 INFO:journalctl@ceph.osd.1.smithi037.stdout:Sep 20 22:25:03 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-1-activate[74004]: --> ceph-volume raw activate successful for osd ID: 1 2024-09-20T22:25:03.402 INFO:journalctl@ceph.osd.1.smithi037.stdout:Sep 20 22:25:03 smithi037 bash[73992]: --> ceph-volume raw activate successful for osd ID: 1 2024-09-20T22:25:03.402 INFO:journalctl@ceph.osd.1.smithi037.stdout:Sep 20 22:25:03 smithi037 podman[73992]: 2024-09-20 22:25:03.316694234 +0000 UTC m=+2.548917801 container died 60bfeb71df3b549f5928890e3b6a455fc1d54003d3c45253f7c6efb630d741a7 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-1-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-9e2ac70, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.build-date=20240731) 2024-09-20T22:25:03.854 INFO:teuthology.orchestra.run.smithi037.stdout: 5 184773 10.76 MB/sec execute 321 sec latency 65.036 ms 2024-09-20T22:25:04.184 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:04 smithi037 ceph-mon[65640]: Health check update: Degraded data redundancy: 179/1578 objects degraded (11.343%), 23 pgs degraded (PG_DEGRADED) 2024-09-20T22:25:04.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:04 smithi194 ceph-mon[51725]: Health check update: Degraded data redundancy: 179/1578 objects degraded (11.343%), 23 pgs degraded (PG_DEGRADED) 2024-09-20T22:25:04.437 INFO:journalctl@ceph.osd.1.smithi037.stdout:Sep 20 22:25:04 smithi037 podman[74154]: 2024-09-20 22:25:04.280891705 +0000 UTC m=+0.954739516 container remove 60bfeb71df3b549f5928890e3b6a455fc1d54003d3c45253f7c6efb630d741a7 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-1-activate, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, RELEASE=squid-9e2ac70, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-09-20T22:25:04.688 INFO:journalctl@ceph.osd.1.smithi037.stdout:Sep 20 22:25:04 smithi037 podman[74212]: 2024-09-20 22:25:04.431997487 +0000 UTC m=+0.021367641 image pull cc9251b744a3faafca7c3095f79bfff81873afe823f31a03a11e015a34abcdbd quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5 2024-09-20T22:25:04.855 INFO:teuthology.orchestra.run.smithi037.stdout: 5 185048 10.76 MB/sec execute 322 sec latency 62.292 ms 2024-09-20T22:25:05.209 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:05 smithi037 ceph-mon[65640]: pgmap v96: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 331 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 5.5 MiB/s wr, 586 op/s; 179/1578 objects degraded (11.343%) 2024-09-20T22:25:05.209 INFO:journalctl@ceph.osd.1.smithi037.stdout:Sep 20 22:25:05 smithi037 podman[74212]: 2024-09-20 22:25:05.016655788 +0000 UTC m=+0.606025924 container create f6b715391469057d4b1bfd72f6271784f6be28201e027a8bc5421437e9d85474 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-1, org.label-schema.build-date=20240731, RELEASE=squid-9e2ac70, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_BRANCH=HEAD) 2024-09-20T22:25:05.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:05 smithi194 ceph-mon[51725]: pgmap v96: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 331 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 5.5 MiB/s wr, 586 op/s; 179/1578 objects degraded (11.343%) 2024-09-20T22:25:05.481 INFO:journalctl@ceph.osd.1.smithi037.stdout:Sep 20 22:25:05 smithi037 podman[74212]: 2024-09-20 22:25:05.395419837 +0000 UTC m=+0.984789979 container init f6b715391469057d4b1bfd72f6271784f6be28201e027a8bc5421437e9d85474 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-1, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=squid-9e2ac70) 2024-09-20T22:25:05.481 INFO:journalctl@ceph.osd.1.smithi037.stdout:Sep 20 22:25:05 smithi037 podman[74212]: 2024-09-20 22:25:05.404588475 +0000 UTC m=+0.993958610 container start f6b715391469057d4b1bfd72f6271784f6be28201e027a8bc5421437e9d85474 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-1, maintainer=Guillaume Abrioux , RELEASE=squid-9e2ac70, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-09-20T22:25:05.481 INFO:journalctl@ceph.osd.1.smithi037.stdout:Sep 20 22:25:05 smithi037 ceph-osd[74231]: -- 172.21.15.37:0/1708244023 <== mon.0 v2:172.21.15.37:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55d2856981e0 con 0x55d284885c00 2024-09-20T22:25:05.737 INFO:journalctl@ceph.osd.1.smithi037.stdout:Sep 20 22:25:05 smithi037 bash[74212]: f6b715391469057d4b1bfd72f6271784f6be28201e027a8bc5421437e9d85474 2024-09-20T22:25:05.737 INFO:journalctl@ceph.osd.1.smithi037.stdout:Sep 20 22:25:05 smithi037 systemd[1]: Started Ceph osd.1 for f9a6357a-779c-11ef-baf5-efdc52797490. 2024-09-20T22:25:05.854 INFO:teuthology.orchestra.run.smithi037.stdout: 5 185276 10.75 MB/sec execute 323 sec latency 76.115 ms 2024-09-20T22:25:06.761 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:06 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:25:06.762 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:06 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:25:06.762 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:06 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:25:06.762 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:06 smithi037 ceph-mon[65640]: pgmap v97: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 333 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.1 MiB/s wr, 499 op/s; 181/1596 objects degraded (11.341%) 2024-09-20T22:25:06.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:06 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:25:06.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:06 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:25:06.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:06 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:25:06.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:06 smithi194 ceph-mon[51725]: pgmap v97: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 333 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.1 MiB/s wr, 499 op/s; 181/1596 objects degraded (11.341%) 2024-09-20T22:25:06.854 INFO:teuthology.orchestra.run.smithi037.stdout: 5 185626 10.74 MB/sec execute 324 sec latency 67.427 ms 2024-09-20T22:25:07.525 INFO:journalctl@ceph.osd.1.smithi037.stdout:Sep 20 22:25:07 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-1[74227]: 2024-09-20T22:25:07.517+0000 7faaa1764640 -1 Falling back to public interface 2024-09-20T22:25:07.854 INFO:teuthology.orchestra.run.smithi037.stdout: 5 185907 10.73 MB/sec execute 325 sec latency 80.601 ms 2024-09-20T22:25:08.031 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:07 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:25:08.032 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:07 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:25:08.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:07 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:25:08.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:07 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:25:08.855 INFO:teuthology.orchestra.run.smithi037.stdout: 5 186190 10.71 MB/sec execute 326 sec latency 72.686 ms 2024-09-20T22:25:09.048 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:08 smithi037 ceph-mon[65640]: pgmap v98: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 339 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.3 MiB/s wr, 628 op/s; 185/1632 objects degraded (11.336%) 2024-09-20T22:25:09.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:08 smithi194 ceph-mon[51725]: pgmap v98: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 339 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.3 MiB/s wr, 628 op/s; 185/1632 objects degraded (11.336%) 2024-09-20T22:25:09.855 INFO:teuthology.orchestra.run.smithi037.stdout: 5 186506 10.70 MB/sec execute 327 sec latency 58.831 ms 2024-09-20T22:25:10.082 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:09 smithi037 ceph-mon[65640]: Health check update: Degraded data redundancy: 185/1632 objects degraded (11.336%), 23 pgs degraded (PG_DEGRADED) 2024-09-20T22:25:10.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:09 smithi194 ceph-mon[51725]: Health check update: Degraded data redundancy: 185/1632 objects degraded (11.336%), 23 pgs degraded (PG_DEGRADED) 2024-09-20T22:25:10.857 INFO:teuthology.orchestra.run.smithi037.stdout: 5 186866 10.69 MB/sec execute 328 sec latency 43.290 ms 2024-09-20T22:25:11.109 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:10 smithi037 ceph-mon[65640]: pgmap v99: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 334 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.5 MiB/s wr, 421 op/s; 186/1644 objects degraded (11.314%) 2024-09-20T22:25:11.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:10 smithi194 ceph-mon[51725]: pgmap v99: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 334 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.5 MiB/s wr, 421 op/s; 186/1644 objects degraded (11.314%) 2024-09-20T22:25:11.857 INFO:teuthology.orchestra.run.smithi037.stdout: 5 187253 10.69 MB/sec execute 329 sec latency 36.612 ms 2024-09-20T22:25:12.857 INFO:teuthology.orchestra.run.smithi037.stdout: 5 187533 10.69 MB/sec execute 330 sec latency 83.865 ms 2024-09-20T22:25:13.161 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:12 smithi037 ceph-mon[65640]: pgmap v100: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 334 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.1 MiB/s wr, 402 op/s; 186/1644 objects degraded (11.314%) 2024-09-20T22:25:13.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:12 smithi194 ceph-mon[51725]: pgmap v100: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 334 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.1 MiB/s wr, 402 op/s; 186/1644 objects degraded (11.314%) 2024-09-20T22:25:13.857 INFO:teuthology.orchestra.run.smithi037.stdout: 5 187874 10.68 MB/sec execute 331 sec latency 68.663 ms 2024-09-20T22:25:14.857 INFO:teuthology.orchestra.run.smithi037.stdout: 5 188147 10.67 MB/sec execute 332 sec latency 52.894 ms 2024-09-20T22:25:14.978 INFO:journalctl@ceph.osd.1.smithi037.stdout:Sep 20 22:25:14 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-1[74227]: 2024-09-20T22:25:14.804+0000 7faaa1764640 -1 osd.1 0 read_superblock omap replica is missing. 2024-09-20T22:25:14.978 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:14 smithi037 ceph-mon[65640]: pgmap v101: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 326 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 5.6 MiB/s wr, 594 op/s; 167/1527 objects degraded (10.936%) 2024-09-20T22:25:14.978 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:14 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:25:14.979 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:14 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:25:14.979 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:14 smithi037 ceph-mon[65640]: Health check update: Degraded data redundancy: 167/1527 objects degraded (10.936%), 23 pgs degraded (PG_DEGRADED) 2024-09-20T22:25:15.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:14 smithi194 ceph-mon[51725]: pgmap v101: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 326 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 5.6 MiB/s wr, 594 op/s; 167/1527 objects degraded (10.936%) 2024-09-20T22:25:15.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:14 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:25:15.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:14 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:25:15.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:14 smithi194 ceph-mon[51725]: Health check update: Degraded data redundancy: 167/1527 objects degraded (10.936%), 23 pgs degraded (PG_DEGRADED) 2024-09-20T22:25:15.857 INFO:teuthology.orchestra.run.smithi037.stdout: 5 188441 10.67 MB/sec execute 333 sec latency 116.126 ms 2024-09-20T22:25:16.008 INFO:journalctl@ceph.osd.1.smithi037.stdout:Sep 20 22:25:15 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-1[74227]: 2024-09-20T22:25:15.772+0000 7faaa1764640 -1 osd.1 83 log_to_monitors true 2024-09-20T22:25:16.263 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:16 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:25:16.263 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:16 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:25:16.263 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:16 smithi037 ceph-mon[65640]: from='osd.1 [v2:172.21.15.37:6810/2260387438,v1:172.21.15.37:6811/2260387438]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-20T22:25:16.263 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:16 smithi037 ceph-mon[65640]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-20T22:25:16.264 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:16 smithi037 ceph-mon[65640]: pgmap v102: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 326 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.2 MiB/s wr, 402 op/s; 168/1533 objects degraded (10.959%) 2024-09-20T22:25:16.524 INFO:journalctl@ceph.osd.1.smithi037.stdout:Sep 20 22:25:16 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-1[74227]: 2024-09-20T22:25:16.303+0000 7faa9950f640 -1 osd.1 83 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-20T22:25:16.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:16 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:25:16.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:16 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:25:16.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:16 smithi194 ceph-mon[51725]: from='osd.1 [v2:172.21.15.37:6810/2260387438,v1:172.21.15.37:6811/2260387438]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-20T22:25:16.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:16 smithi194 ceph-mon[51725]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-20T22:25:16.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:16 smithi194 ceph-mon[51725]: pgmap v102: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 326 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.2 MiB/s wr, 402 op/s; 168/1533 objects degraded (10.959%) 2024-09-20T22:25:16.857 INFO:teuthology.orchestra.run.smithi037.stdout: 5 188771 10.67 MB/sec execute 334 sec latency 57.207 ms 2024-09-20T22:25:17.282 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:17 smithi037 ceph-mon[65640]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-20T22:25:17.282 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:17 smithi037 ceph-mon[65640]: from='osd.1 [v2:172.21.15.37:6810/2260387438,v1:172.21.15.37:6811/2260387438]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi037", "root=default"]}]: dispatch 2024-09-20T22:25:17.282 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:17 smithi037 ceph-mon[65640]: osdmap e86: 8 total, 7 up, 8 in 2024-09-20T22:25:17.282 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:17 smithi037 ceph-mon[65640]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi037", "root=default"]}]: dispatch 2024-09-20T22:25:17.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:17 smithi194 ceph-mon[51725]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-20T22:25:17.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:17 smithi194 ceph-mon[51725]: from='osd.1 [v2:172.21.15.37:6810/2260387438,v1:172.21.15.37:6811/2260387438]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi037", "root=default"]}]: dispatch 2024-09-20T22:25:17.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:17 smithi194 ceph-mon[51725]: osdmap e86: 8 total, 7 up, 8 in 2024-09-20T22:25:17.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:17 smithi194 ceph-mon[51725]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi037", "root=default"]}]: dispatch 2024-09-20T22:25:17.857 INFO:teuthology.orchestra.run.smithi037.stdout: 5 188934 10.65 MB/sec execute 335 sec latency 617.222 ms 2024-09-20T22:25:18.331 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:18 smithi037 ceph-mon[65640]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-20T22:25:18.331 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:18 smithi037 ceph-mon[65640]: osd.1 [v2:172.21.15.37:6810/2260387438,v1:172.21.15.37:6811/2260387438] boot 2024-09-20T22:25:18.331 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:18 smithi037 ceph-mon[65640]: osdmap e87: 8 total, 8 up, 8 in 2024-09-20T22:25:18.331 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:18 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-20T22:25:18.331 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:18 smithi037 ceph-mon[65640]: pgmap v105: 97 pgs: 20 peering, 6 active+undersized, 13 active+undersized+degraded, 58 active+clean; 334 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 4.4 MiB/s wr, 432 op/s; 94/1599 objects degraded (5.879%) 2024-09-20T22:25:18.331 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:18 smithi037 ceph-mon[65640]: osdmap e88: 8 total, 8 up, 8 in 2024-09-20T22:25:18.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:18 smithi194 ceph-mon[51725]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-20T22:25:18.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:18 smithi194 ceph-mon[51725]: osd.1 [v2:172.21.15.37:6810/2260387438,v1:172.21.15.37:6811/2260387438] boot 2024-09-20T22:25:18.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:18 smithi194 ceph-mon[51725]: osdmap e87: 8 total, 8 up, 8 in 2024-09-20T22:25:18.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:18 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-20T22:25:18.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:18 smithi194 ceph-mon[51725]: pgmap v105: 97 pgs: 20 peering, 6 active+undersized, 13 active+undersized+degraded, 58 active+clean; 334 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 4.4 MiB/s wr, 432 op/s; 94/1599 objects degraded (5.879%) 2024-09-20T22:25:18.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:18 smithi194 ceph-mon[51725]: osdmap e88: 8 total, 8 up, 8 in 2024-09-20T22:25:18.857 INFO:teuthology.orchestra.run.smithi037.stdout: 5 189090 10.62 MB/sec execute 336 sec latency 1617.257 ms 2024-09-20T22:25:19.857 INFO:teuthology.orchestra.run.smithi037.stdout: 5 189228 10.60 MB/sec execute 337 sec latency 2056.927 ms 2024-09-20T22:25:20.314 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:20 smithi037 ceph-mon[65640]: osdmap e89: 8 total, 8 up, 8 in 2024-09-20T22:25:20.314 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:20 smithi037 ceph-mon[65640]: Health check update: Degraded data redundancy: 94/1599 objects degraded (5.879%), 13 pgs degraded (PG_DEGRADED) 2024-09-20T22:25:20.314 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:20 smithi037 ceph-mon[65640]: osdmap e90: 8 total, 8 up, 8 in 2024-09-20T22:25:20.315 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:20 smithi037 ceph-mon[65640]: pgmap v109: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 23 peering, 5 active+undersized, 9 active+undersized+degraded, 59 active+clean; 335 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 6.4 MiB/s wr, 534 op/s; 81/1641 objects degraded (4.936%) 2024-09-20T22:25:20.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:20 smithi194 ceph-mon[51725]: osdmap e89: 8 total, 8 up, 8 in 2024-09-20T22:25:20.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:20 smithi194 ceph-mon[51725]: Health check update: Degraded data redundancy: 94/1599 objects degraded (5.879%), 13 pgs degraded (PG_DEGRADED) 2024-09-20T22:25:20.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:20 smithi194 ceph-mon[51725]: osdmap e90: 8 total, 8 up, 8 in 2024-09-20T22:25:20.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:20 smithi194 ceph-mon[51725]: pgmap v109: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 23 peering, 5 active+undersized, 9 active+undersized+degraded, 59 active+clean; 335 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 6.4 MiB/s wr, 534 op/s; 81/1641 objects degraded (4.936%) 2024-09-20T22:25:20.857 INFO:teuthology.orchestra.run.smithi037.stdout: 5 189614 10.62 MB/sec execute 338 sec latency 482.421 ms 2024-09-20T22:25:21.570 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:21 smithi037 ceph-mon[65640]: osdmap e91: 8 total, 8 up, 8 in 2024-09-20T22:25:21.570 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:21 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:25:21.570 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:21 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:25:21.570 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:21 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:25:21.570 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:21 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:25:21.570 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:21 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:25:21.570 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:21 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:25:21.826 INFO:teuthology.orchestra.run.smithi037.stdout:true 2024-09-20T22:25:21.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:21 smithi194 ceph-mon[51725]: osdmap e91: 8 total, 8 up, 8 in 2024-09-20T22:25:21.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:21 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:25:21.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:21 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:25:21.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:21 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:25:21.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:21 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:25:21.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:21 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:25:21.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:21 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:25:21.857 INFO:teuthology.orchestra.run.smithi037.stdout: 5 189934 10.60 MB/sec execute 339 sec latency 299.927 ms 2024-09-20T22:25:22.592 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:22 smithi037 ceph-mon[65640]: osdmap e92: 8 total, 8 up, 8 in 2024-09-20T22:25:22.592 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:22 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:25:22.592 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:22 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:25:22.593 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:22 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:25:22.593 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:22 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:25:22.593 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:22 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-20T22:25:22.593 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:22 smithi037 ceph-mon[65640]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-20T22:25:22.593 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:22 smithi037 ceph-mon[65640]: Upgrade: unsafe to stop osd(s) at this time (5 PGs are or would become offline) 2024-09-20T22:25:22.593 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:22 smithi037 ceph-mon[65640]: from='client.44187 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:25:22.593 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:22 smithi037 ceph-mon[65640]: pgmap v112: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 23 peering, 3 active+undersized+degraded, 70 active+clean; 335 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 876 KiB/s wr, 34 op/s; 61/1641 objects degraded (3.717%); 24 B/s, 76 keys/s, 0 objects/s recovering 2024-09-20T22:25:22.593 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:22 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:25:22.593 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:22 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:25:22.593 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:22 smithi037 ceph-mon[65640]: osdmap e93: 8 total, 8 up, 8 in 2024-09-20T22:25:22.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:22 smithi194 ceph-mon[51725]: osdmap e92: 8 total, 8 up, 8 in 2024-09-20T22:25:22.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:22 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:25:22.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:22 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:25:22.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:22 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:25:22.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:22 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:25:22.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:22 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-20T22:25:22.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:22 smithi194 ceph-mon[51725]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-20T22:25:22.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:22 smithi194 ceph-mon[51725]: Upgrade: unsafe to stop osd(s) at this time (5 PGs are or would become offline) 2024-09-20T22:25:22.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:22 smithi194 ceph-mon[51725]: from='client.44187 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:25:22.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:22 smithi194 ceph-mon[51725]: pgmap v112: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 23 peering, 3 active+undersized+degraded, 70 active+clean; 335 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 876 KiB/s wr, 34 op/s; 61/1641 objects degraded (3.717%); 24 B/s, 76 keys/s, 0 objects/s recovering 2024-09-20T22:25:22.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:22 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:25:22.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:22 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:25:22.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:22 smithi194 ceph-mon[51725]: osdmap e93: 8 total, 8 up, 8 in 2024-09-20T22:25:22.857 INFO:teuthology.orchestra.run.smithi037.stdout: 5 190203 10.58 MB/sec execute 340 sec latency 953.500 ms 2024-09-20T22:25:23.021 INFO:teuthology.orchestra.run.smithi037.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T22:25:23.022 INFO:teuthology.orchestra.run.smithi037.stdout:alertmanager.smithi037 smithi037 *:9093,9094 running (5m) 8s ago 14m 25.8M - 0.25.0 c8568f914cd2 d1f99b3cd707 2024-09-20T22:25:23.022 INFO:teuthology.orchestra.run.smithi037.stdout:crash.smithi037 smithi037 running (111s) 8s ago 14m 6895k - 19.1.1-367-g9e2ac70b cc9251b744a3 77e510d8bdb1 2024-09-20T22:25:23.022 INFO:teuthology.orchestra.run.smithi037.stdout:crash.smithi194 smithi194 running (105s) 101s ago 13m 6895k - 19.1.1-367-g9e2ac70b cc9251b744a3 4d169d1309bc 2024-09-20T22:25:23.022 INFO:teuthology.orchestra.run.smithi037.stdout:grafana.smithi037 smithi037 *:3000 running (5m) 8s ago 13m 54.2M - dad864ee21e9 2306f3c2da19 2024-09-20T22:25:23.022 INFO:teuthology.orchestra.run.smithi037.stdout:mds.foofs.smithi037.sdtedi smithi037 running (8m) 8s ago 8m 38.0M - 17.2.0 e1d6a67b021e 0a7898d25b8d 2024-09-20T22:25:23.022 INFO:teuthology.orchestra.run.smithi037.stdout:mds.foofs.smithi194.lkroso smithi194 running (8m) 101s ago 8m 15.6M - 17.2.0 e1d6a67b021e 17736de30008 2024-09-20T22:25:23.022 INFO:teuthology.orchestra.run.smithi037.stdout:mgr.smithi037.lykblh smithi037 *:8443,9283,8765 running (4m) 8s ago 15m 502M - 19.1.1-367-g9e2ac70b cc9251b744a3 181e46fd9ba6 2024-09-20T22:25:23.022 INFO:teuthology.orchestra.run.smithi037.stdout:mgr.smithi194.bkcglt smithi194 *:8443,9283,8765 running (3m) 101s ago 13m 439M - 19.1.1-367-g9e2ac70b cc9251b744a3 1fa610c97517 2024-09-20T22:25:23.022 INFO:teuthology.orchestra.run.smithi037.stdout:mon.smithi037 smithi037 running (3m) 8s ago 15m 48.5M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 c0229eceb527 2024-09-20T22:25:23.022 INFO:teuthology.orchestra.run.smithi037.stdout:mon.smithi194 smithi194 running (2m) 101s ago 13m 37.2M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 97854807ba91 2024-09-20T22:25:23.022 INFO:teuthology.orchestra.run.smithi037.stdout:nfs.foo.0.0.smithi037.spabdc smithi037 *:2049 running (7m) 8s ago 7m 268M - 4.0 e1d6a67b021e 858ff9d558a2 2024-09-20T22:25:23.022 INFO:teuthology.orchestra.run.smithi037.stdout:nfs.foo.1.0.smithi194.qzjyej smithi194 *:2049 running (7m) 101s ago 7m 89.8M - 4.0 e1d6a67b021e c07fe0eb7c75 2024-09-20T22:25:23.022 INFO:teuthology.orchestra.run.smithi037.stdout:node-exporter.smithi037 smithi037 *:9100 running (5m) 8s ago 13m 16.3M - 1.5.0 0da6a335fe13 433e6aaf6137 2024-09-20T22:25:23.023 INFO:teuthology.orchestra.run.smithi037.stdout:node-exporter.smithi194 smithi194 *:9100 running (4m) 101s ago 13m 15.7M - 1.5.0 0da6a335fe13 1d30bb810ad1 2024-09-20T22:25:23.023 INFO:teuthology.orchestra.run.smithi037.stdout:osd.0 smithi037 running (69s) 8s ago 12m 240M 1937M 19.1.1-367-g9e2ac70b cc9251b744a3 c5568bd43c07 2024-09-20T22:25:23.023 INFO:teuthology.orchestra.run.smithi037.stdout:osd.1 smithi037 running (18s) 8s ago 12m 11.5M 1937M 19.1.1-367-g9e2ac70b cc9251b744a3 f6b715391469 2024-09-20T22:25:23.023 INFO:teuthology.orchestra.run.smithi037.stdout:osd.2 smithi037 running (11m) 8s ago 11m 470M 1937M 17.2.0 e1d6a67b021e 6c15b1175256 2024-09-20T22:25:23.023 INFO:teuthology.orchestra.run.smithi037.stdout:osd.3 smithi037 running (11m) 8s ago 11m 587M 1937M 17.2.0 e1d6a67b021e 3a22952c6475 2024-09-20T22:25:23.023 INFO:teuthology.orchestra.run.smithi037.stdout:osd.4 smithi194 running (10m) 101s ago 10m 513M 2704M 17.2.0 e1d6a67b021e fa107be8915c 2024-09-20T22:25:23.023 INFO:teuthology.orchestra.run.smithi037.stdout:osd.5 smithi194 running (10m) 101s ago 10m 324M 2704M 17.2.0 e1d6a67b021e 290b21cbd131 2024-09-20T22:25:23.023 INFO:teuthology.orchestra.run.smithi037.stdout:osd.6 smithi194 running (10m) 101s ago 10m 511M 2704M 17.2.0 e1d6a67b021e 27f0720639ed 2024-09-20T22:25:23.023 INFO:teuthology.orchestra.run.smithi037.stdout:osd.7 smithi194 running (9m) 101s ago 9m 406M 2704M 17.2.0 e1d6a67b021e 62d163f4fd92 2024-09-20T22:25:23.023 INFO:teuthology.orchestra.run.smithi037.stdout:prometheus.smithi037 smithi037 *:9095 running (3m) 8s ago 13m 63.8M - 2.43.0 a07b618ecd1d 05b826dde2ff 2024-09-20T22:25:23.569 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:23 smithi037 ceph-mon[65640]: from='client.34254 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:25:23.569 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:23 smithi037 ceph-mon[65640]: from='client.44193 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:25:23.569 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:23 smithi037 ceph-mon[65640]: osdmap e94: 8 total, 8 up, 8 in 2024-09-20T22:25:23.669 INFO:teuthology.orchestra.run.smithi037.stdout:{ 2024-09-20T22:25:23.669 INFO:teuthology.orchestra.run.smithi037.stdout: "mon": { 2024-09-20T22:25:23.670 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 2 2024-09-20T22:25:23.670 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:25:23.670 INFO:teuthology.orchestra.run.smithi037.stdout: "mgr": { 2024-09-20T22:25:23.670 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 2 2024-09-20T22:25:23.670 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:25:23.670 INFO:teuthology.orchestra.run.smithi037.stdout: "osd": { 2024-09-20T22:25:23.670 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-09-20T22:25:23.670 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 2 2024-09-20T22:25:23.670 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:25:23.670 INFO:teuthology.orchestra.run.smithi037.stdout: "mds": { 2024-09-20T22:25:23.671 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T22:25:23.671 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:25:23.671 INFO:teuthology.orchestra.run.smithi037.stdout: "overall": { 2024-09-20T22:25:23.671 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8, 2024-09-20T22:25:23.671 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 6 2024-09-20T22:25:23.671 INFO:teuthology.orchestra.run.smithi037.stdout: } 2024-09-20T22:25:23.671 INFO:teuthology.orchestra.run.smithi037.stdout:} 2024-09-20T22:25:23.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:23 smithi194 ceph-mon[51725]: from='client.34254 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:25:23.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:23 smithi194 ceph-mon[51725]: from='client.44193 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:25:23.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:23 smithi194 ceph-mon[51725]: osdmap e94: 8 total, 8 up, 8 in 2024-09-20T22:25:23.857 INFO:teuthology.orchestra.run.smithi037.stdout: 5 190492 10.56 MB/sec execute 341 sec latency 600.717 ms 2024-09-20T22:25:24.131 INFO:teuthology.orchestra.run.smithi037.stdout:{ 2024-09-20T22:25:24.131 INFO:teuthology.orchestra.run.smithi037.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5", 2024-09-20T22:25:24.131 INFO:teuthology.orchestra.run.smithi037.stdout: "in_progress": true, 2024-09-20T22:25:24.131 INFO:teuthology.orchestra.run.smithi037.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-20T22:25:24.131 INFO:teuthology.orchestra.run.smithi037.stdout: "services_complete": [ 2024-09-20T22:25:24.131 INFO:teuthology.orchestra.run.smithi037.stdout: "mon", 2024-09-20T22:25:24.131 INFO:teuthology.orchestra.run.smithi037.stdout: "mgr", 2024-09-20T22:25:24.131 INFO:teuthology.orchestra.run.smithi037.stdout: "crash" 2024-09-20T22:25:24.131 INFO:teuthology.orchestra.run.smithi037.stdout: ], 2024-09-20T22:25:24.132 INFO:teuthology.orchestra.run.smithi037.stdout: "progress": "8/23 daemons upgraded", 2024-09-20T22:25:24.132 INFO:teuthology.orchestra.run.smithi037.stdout: "message": "Currently upgrading osd daemons", 2024-09-20T22:25:24.132 INFO:teuthology.orchestra.run.smithi037.stdout: "is_paused": false 2024-09-20T22:25:24.132 INFO:teuthology.orchestra.run.smithi037.stdout:} 2024-09-20T22:25:24.615 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:24 smithi037 ceph-mon[65640]: from='client.? 172.21.15.37:0/3971884167' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:25:24.615 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:24 smithi037 ceph-mon[65640]: pgmap v115: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 peering, 1 active+undersized+degraded, 90 active+clean; 335 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 3.4 MiB/s wr, 457 op/s; 108/1593 objects degraded (6.780%); 1024 KiB/s, 68 keys/s, 35 objects/s recovering 2024-09-20T22:25:24.615 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:24 smithi037 ceph-mon[65640]: from='client.34264 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:25:24.615 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:24 smithi037 ceph-mon[65640]: Health check update: Degraded data redundancy: 61/1641 objects degraded (3.717%), 4 pgs degraded (PG_DEGRADED) 2024-09-20T22:25:24.615 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:24 smithi037 ceph-mon[65640]: osdmap e95: 8 total, 8 up, 8 in 2024-09-20T22:25:24.669 INFO:teuthology.orchestra.run.smithi037.stdout:HEALTH_WARN Degraded data redundancy: 108/1593 objects degraded (6.780%), 5 pgs degraded 2024-09-20T22:25:24.669 INFO:teuthology.orchestra.run.smithi037.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 108/1593 objects degraded (6.780%), 5 pgs degraded 2024-09-20T22:25:24.669 INFO:teuthology.orchestra.run.smithi037.stdout: pg 3.0 is active+undersized+degraded, acting [2,6] 2024-09-20T22:25:24.669 INFO:teuthology.orchestra.run.smithi037.stdout: pg 3.4 is active+recovery_wait+undersized+degraded+remapped, acting [2,5] 2024-09-20T22:25:24.669 INFO:teuthology.orchestra.run.smithi037.stdout: pg 3.6 is active+recovery_wait+undersized+degraded+remapped, acting [0,4] 2024-09-20T22:25:24.670 INFO:teuthology.orchestra.run.smithi037.stdout: pg 3.16 is active+recovering+undersized+degraded+remapped, acting [5,7] 2024-09-20T22:25:24.670 INFO:teuthology.orchestra.run.smithi037.stdout: pg 3.19 is active+recovery_wait+undersized+degraded+remapped, acting [3,4] 2024-09-20T22:25:24.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:24 smithi194 ceph-mon[51725]: from='client.? 172.21.15.37:0/3971884167' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:25:24.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:24 smithi194 ceph-mon[51725]: pgmap v115: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 peering, 1 active+undersized+degraded, 90 active+clean; 335 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 3.4 MiB/s wr, 457 op/s; 108/1593 objects degraded (6.780%); 1024 KiB/s, 68 keys/s, 35 objects/s recovering 2024-09-20T22:25:24.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:24 smithi194 ceph-mon[51725]: from='client.34264 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:25:24.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:24 smithi194 ceph-mon[51725]: Health check update: Degraded data redundancy: 61/1641 objects degraded (3.717%), 4 pgs degraded (PG_DEGRADED) 2024-09-20T22:25:24.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:24 smithi194 ceph-mon[51725]: osdmap e95: 8 total, 8 up, 8 in 2024-09-20T22:25:24.857 INFO:teuthology.orchestra.run.smithi037.stdout: 5 190797 10.56 MB/sec execute 342 sec latency 798.098 ms 2024-09-20T22:25:25.624 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:25 smithi037 ceph-mon[65640]: from='client.? 172.21.15.37:0/3495140396' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T22:25:25.624 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:25 smithi037 ceph-mon[65640]: osdmap e96: 8 total, 8 up, 8 in 2024-09-20T22:25:25.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:25 smithi194 ceph-mon[51725]: from='client.? 172.21.15.37:0/3495140396' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T22:25:25.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:25 smithi194 ceph-mon[51725]: osdmap e96: 8 total, 8 up, 8 in 2024-09-20T22:25:25.857 INFO:teuthology.orchestra.run.smithi037.stdout: 5 191156 10.56 MB/sec execute 343 sec latency 675.156 ms 2024-09-20T22:25:26.645 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:26 smithi037 ceph-mon[65640]: pgmap v118: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 peering, 1 active+undersized+degraded, 91 active+clean; 335 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.3 MiB/s wr, 501 op/s; 81/1551 objects degraded (5.222%); 1.1 MiB/s, 41 objects/s recovering 2024-09-20T22:25:26.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:26 smithi194 ceph-mon[51725]: pgmap v118: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 peering, 1 active+undersized+degraded, 91 active+clean; 335 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.3 MiB/s wr, 501 op/s; 81/1551 objects degraded (5.222%); 1.1 MiB/s, 41 objects/s recovering 2024-09-20T22:25:26.857 INFO:teuthology.orchestra.run.smithi037.stdout: 5 191593 10.57 MB/sec execute 344 sec latency 39.600 ms 2024-09-20T22:25:27.857 INFO:teuthology.orchestra.run.smithi037.stdout: 5 191945 10.58 MB/sec execute 345 sec latency 47.842 ms 2024-09-20T22:25:28.191 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:27 smithi037 ceph-mon[65640]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 81/1551 objects degraded (5.222%), 4 pgs degraded) 2024-09-20T22:25:28.191 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:27 smithi037 ceph-mon[65640]: Cluster is now healthy 2024-09-20T22:25:28.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:27 smithi194 ceph-mon[51725]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 81/1551 objects degraded (5.222%), 4 pgs degraded) 2024-09-20T22:25:28.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:27 smithi194 ceph-mon[51725]: Cluster is now healthy 2024-09-20T22:25:28.857 INFO:teuthology.orchestra.run.smithi037.stdout: 5 192478 10.59 MB/sec execute 346 sec latency 59.050 ms 2024-09-20T22:25:29.157 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:28 smithi037 ceph-mon[65640]: pgmap v119: 97 pgs: 97 active+clean; 334 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 7.6 MiB/s wr, 811 op/s; 871 KiB/s, 48 objects/s recovering 2024-09-20T22:25:29.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:28 smithi194 ceph-mon[51725]: pgmap v119: 97 pgs: 97 active+clean; 334 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 7.6 MiB/s wr, 811 op/s; 871 KiB/s, 48 objects/s recovering 2024-09-20T22:25:29.857 INFO:teuthology.orchestra.run.smithi037.stdout: 5 192968 10.60 MB/sec execute 347 sec latency 36.714 ms 2024-09-20T22:25:30.857 INFO:teuthology.orchestra.run.smithi037.stdout: 5 193502 10.60 MB/sec execute 348 sec latency 39.986 ms 2024-09-20T22:25:31.208 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:30 smithi037 ceph-mon[65640]: pgmap v120: 97 pgs: 97 active+clean; 335 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 5.0 MiB/s wr, 426 op/s; 90 KiB/s, 18 objects/s recovering 2024-09-20T22:25:31.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:30 smithi194 ceph-mon[51725]: pgmap v120: 97 pgs: 97 active+clean; 335 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 5.0 MiB/s wr, 426 op/s; 90 KiB/s, 18 objects/s recovering 2024-09-20T22:25:31.857 INFO:teuthology.orchestra.run.smithi037.stdout: 5 193921 10.59 MB/sec execute 349 sec latency 48.416 ms 2024-09-20T22:25:32.858 INFO:teuthology.orchestra.run.smithi037.stdout: 5 194346 10.59 MB/sec execute 350 sec latency 34.298 ms 2024-09-20T22:25:32.999 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:32 smithi037 ceph-mon[65640]: pgmap v121: 97 pgs: 97 active+clean; 338 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.2 KiB/s rd, 4.7 MiB/s wr, 476 op/s; 72 KiB/s, 14 objects/s recovering 2024-09-20T22:25:33.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:32 smithi194 ceph-mon[51725]: pgmap v121: 97 pgs: 97 active+clean; 338 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.2 KiB/s rd, 4.7 MiB/s wr, 476 op/s; 72 KiB/s, 14 objects/s recovering 2024-09-20T22:25:33.857 INFO:teuthology.orchestra.run.smithi037.stdout: 5 194836 10.60 MB/sec execute 351 sec latency 29.533 ms 2024-09-20T22:25:34.857 INFO:teuthology.orchestra.run.smithi037.stdout: 5 195296 10.61 MB/sec execute 352 sec latency 35.173 ms 2024-09-20T22:25:35.027 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:34 smithi037 ceph-mon[65640]: pgmap v122: 97 pgs: 97 active+clean; 331 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.1 MiB/s wr, 585 op/s; 61 KiB/s, 12 objects/s recovering 2024-09-20T22:25:35.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:34 smithi194 ceph-mon[51725]: pgmap v122: 97 pgs: 97 active+clean; 331 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.1 MiB/s wr, 585 op/s; 61 KiB/s, 12 objects/s recovering 2024-09-20T22:25:35.858 INFO:teuthology.orchestra.run.smithi037.stdout: 5 195722 10.63 MB/sec execute 353 sec latency 40.161 ms 2024-09-20T22:25:36.857 INFO:teuthology.orchestra.run.smithi037.stdout: 5 196084 10.63 MB/sec execute 354 sec latency 30.013 ms 2024-09-20T22:25:37.042 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:36 smithi037 ceph-mon[65640]: pgmap v123: 97 pgs: 97 active+clean; 337 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 6.3 MiB/s wr, 652 op/s; 0 B/s, 8 objects/s recovering 2024-09-20T22:25:37.042 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:36 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-20T22:25:37.042 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:36 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:25:37.042 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:36 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:25:37.042 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:36 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:25:37.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:36 smithi194 ceph-mon[51725]: pgmap v123: 97 pgs: 97 active+clean; 337 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 6.3 MiB/s wr, 652 op/s; 0 B/s, 8 objects/s recovering 2024-09-20T22:25:37.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:36 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-20T22:25:37.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:36 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:25:37.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:36 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:25:37.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:36 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:25:37.858 INFO:teuthology.orchestra.run.smithi037.stdout: 5 196414 10.62 MB/sec execute 355 sec latency 72.873 ms 2024-09-20T22:25:38.055 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:37 smithi037 ceph-mon[65640]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-20T22:25:38.055 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:37 smithi037 ceph-mon[65640]: Upgrade: osd.2 is safe to restart 2024-09-20T22:25:38.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:37 smithi194 ceph-mon[51725]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-20T22:25:38.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:37 smithi194 ceph-mon[51725]: Upgrade: osd.2 is safe to restart 2024-09-20T22:25:38.858 INFO:teuthology.orchestra.run.smithi037.stdout: 5 196714 10.61 MB/sec execute 356 sec latency 42.908 ms 2024-09-20T22:25:39.079 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:38 smithi037 ceph-mon[65640]: pgmap v124: 97 pgs: 97 active+clean; 337 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.6 MiB/s wr, 736 op/s; 0 B/s, 7 objects/s recovering 2024-09-20T22:25:39.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:38 smithi194 ceph-mon[51725]: pgmap v124: 97 pgs: 97 active+clean; 337 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.6 MiB/s wr, 736 op/s; 0 B/s, 7 objects/s recovering 2024-09-20T22:25:39.857 INFO:teuthology.orchestra.run.smithi037.stdout: 5 197089 10.60 MB/sec execute 357 sec latency 51.402 ms 2024-09-20T22:25:40.348 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:40 smithi037 ceph-mon[65640]: Upgrade: Updating osd.2 2024-09-20T22:25:40.349 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:40 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:25:40.349 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:40 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-20T22:25:40.349 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:40 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:25:40.349 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:40 smithi037 ceph-mon[65640]: Deploying daemon osd.2 on smithi037 2024-09-20T22:25:40.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:40 smithi194 ceph-mon[51725]: Upgrade: Updating osd.2 2024-09-20T22:25:40.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:40 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:25:40.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:40 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-20T22:25:40.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:40 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:25:40.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:40 smithi194 ceph-mon[51725]: Deploying daemon osd.2 on smithi037 2024-09-20T22:25:40.858 INFO:teuthology.orchestra.run.smithi037.stdout: 5 197351 10.58 MB/sec execute 358 sec latency 71.027 ms 2024-09-20T22:25:41.161 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:41 smithi037 ceph-mon[65640]: pgmap v125: 97 pgs: 97 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.9 MiB/s wr, 556 op/s 2024-09-20T22:25:41.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:41 smithi194 ceph-mon[51725]: pgmap v125: 97 pgs: 97 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.9 MiB/s wr, 556 op/s 2024-09-20T22:25:41.858 INFO:teuthology.orchestra.run.smithi037.stdout: 5 197638 10.57 MB/sec execute 359 sec latency 66.501 ms 2024-09-20T22:25:42.858 INFO:teuthology.orchestra.run.smithi037.stdout: 5 197957 10.57 MB/sec execute 360 sec latency 33.616 ms 2024-09-20T22:25:43.250 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:43 smithi037 ceph-mon[65640]: pgmap v126: 97 pgs: 97 active+clean; 331 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.6 MiB/s wr, 548 op/s 2024-09-20T22:25:43.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:43 smithi194 ceph-mon[51725]: pgmap v126: 97 pgs: 97 active+clean; 331 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.6 MiB/s wr, 548 op/s 2024-09-20T22:25:43.858 INFO:teuthology.orchestra.run.smithi037.stdout: 5 198254 10.57 MB/sec execute 361 sec latency 70.779 ms 2024-09-20T22:25:44.858 INFO:teuthology.orchestra.run.smithi037.stdout: 5 198861 10.59 MB/sec execute 362 sec latency 30.041 ms 2024-09-20T22:25:45.297 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:45 smithi037 ceph-mon[65640]: pgmap v127: 97 pgs: 97 active+clean; 327 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.5 MiB/s wr, 634 op/s 2024-09-20T22:25:45.298 INFO:journalctl@ceph.osd.2.smithi037.stdout:Sep 20 22:25:45 smithi037 systemd[1]: Stopping Ceph osd.2 for f9a6357a-779c-11ef-baf5-efdc52797490... 2024-09-20T22:25:45.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:45 smithi194 ceph-mon[51725]: pgmap v127: 97 pgs: 97 active+clean; 327 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.5 MiB/s wr, 634 op/s 2024-09-20T22:25:45.799 INFO:journalctl@ceph.osd.2.smithi037.stdout:Sep 20 22:25:45 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-2[40992]: 2024-09-20T22:25:45.609+0000 7f9fe7d41700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-20T22:25:45.799 INFO:journalctl@ceph.osd.2.smithi037.stdout:Sep 20 22:25:45 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-2[40992]: 2024-09-20T22:25:45.609+0000 7f9fe7d41700 -1 osd.2 96 *** Got signal Terminated *** 2024-09-20T22:25:45.799 INFO:journalctl@ceph.osd.2.smithi037.stdout:Sep 20 22:25:45 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-2[40992]: 2024-09-20T22:25:45.609+0000 7f9fe7d41700 -1 osd.2 96 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-20T22:25:45.858 INFO:teuthology.orchestra.run.smithi037.stdout: 5 199481 10.62 MB/sec execute 363 sec latency 24.875 ms 2024-09-20T22:25:46.301 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:46 smithi037 ceph-mon[65640]: osd.2 marked itself down and dead 2024-09-20T22:25:46.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:46 smithi194 ceph-mon[51725]: osd.2 marked itself down and dead 2024-09-20T22:25:46.858 INFO:teuthology.orchestra.run.smithi037.stdout: 5 199878 10.61 MB/sec execute 364 sec latency 671.062 ms 2024-09-20T22:25:47.163 INFO:journalctl@ceph.osd.2.smithi037.stdout:Sep 20 22:25:46 smithi037 podman[76375]: 2024-09-20 22:25:46.846976544 +0000 UTC m=+1.421694667 container died 6c15b117525669460270665c631cd4614dae52c4b02a39346e6f56ba22c51804 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-2, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., vcs-type=git, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, release=754, GIT_BRANCH=HEAD, distribution-scope=public, io.buildah.version=1.19.8, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, com.redhat.component=centos-stream-container, GIT_CLEAN=True) 2024-09-20T22:25:47.414 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:47 smithi037 ceph-mon[65640]: pgmap v128: 97 pgs: 97 active+clean; 328 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 5.2 MiB/s wr, 511 op/s 2024-09-20T22:25:47.414 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:47 smithi037 ceph-mon[65640]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-20T22:25:47.414 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:47 smithi037 ceph-mon[65640]: osdmap e97: 8 total, 7 up, 8 in 2024-09-20T22:25:47.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:47 smithi194 ceph-mon[51725]: pgmap v128: 97 pgs: 97 active+clean; 328 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 5.2 MiB/s wr, 511 op/s 2024-09-20T22:25:47.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:47 smithi194 ceph-mon[51725]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-20T22:25:47.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:47 smithi194 ceph-mon[51725]: osdmap e97: 8 total, 7 up, 8 in 2024-09-20T22:25:47.858 INFO:teuthology.orchestra.run.smithi037.stdout: 5 200234 10.62 MB/sec execute 365 sec latency 1006.270 ms 2024-09-20T22:25:47.917 INFO:journalctl@ceph.osd.2.smithi037.stdout:Sep 20 22:25:47 smithi037 podman[76375]: 2024-09-20 22:25:47.834114075 +0000 UTC m=+2.408832205 container cleanup 6c15b117525669460270665c631cd4614dae52c4b02a39346e6f56ba22c51804 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-2, version=8, RELEASE=HEAD, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, distribution-scope=public, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, name=centos-stream, vcs-type=git, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-09-20T22:25:47.917 INFO:journalctl@ceph.osd.2.smithi037.stdout:Sep 20 22:25:47 smithi037 bash[76375]: ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-2 2024-09-20T22:25:48.419 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:48 smithi037 ceph-mon[65640]: osdmap e98: 8 total, 7 up, 8 in 2024-09-20T22:25:48.419 INFO:journalctl@ceph.osd.2.smithi037.stdout:Sep 20 22:25:48 smithi037 podman[76391]: 2024-09-20 22:25:48.229331902 +0000 UTC m=+1.376632486 container remove 6c15b117525669460270665c631cd4614dae52c4b02a39346e6f56ba22c51804 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-2, io.k8s.display-name=CentOS Stream 8, name=centos-stream, maintainer=Guillaume Abrioux , vcs-type=git, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public) 2024-09-20T22:25:48.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:48 smithi194 ceph-mon[51725]: osdmap e98: 8 total, 7 up, 8 in 2024-09-20T22:25:48.858 INFO:teuthology.orchestra.run.smithi037.stdout: 5 200794 10.61 MB/sec execute 366 sec latency 46.064 ms 2024-09-20T22:25:48.928 INFO:journalctl@ceph.osd.2.smithi037.stdout:Sep 20 22:25:48 smithi037 podman[76477]: 2024-09-20 22:25:48.766898874 +0000 UTC m=+0.023854940 image pull cc9251b744a3faafca7c3095f79bfff81873afe823f31a03a11e015a34abcdbd quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5 2024-09-20T22:25:48.928 INFO:journalctl@ceph.osd.2.smithi037.stdout:Sep 20 22:25:48 smithi037 podman[76477]: 2024-09-20 22:25:48.880058232 +0000 UTC m=+0.137014285 container create 5f60a1838e85cee3b6fb884222cd5d5f8c87e523f36ab2edb56be3b0f8e36191 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-2-deactivate, org.label-schema.schema-version=1.0, RELEASE=squid-9e2ac70, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240731, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-20T22:25:49.178 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:49 smithi037 ceph-mon[65640]: pgmap v131: 97 pgs: 2 active+undersized, 5 peering, 15 active+undersized+degraded, 75 active+clean; 334 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.4 MiB/s wr, 684 op/s; 101/1587 objects degraded (6.364%) 2024-09-20T22:25:49.179 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:49 smithi037 ceph-mon[65640]: Health check failed: Degraded data redundancy: 101/1587 objects degraded (6.364%), 15 pgs degraded (PG_DEGRADED) 2024-09-20T22:25:49.430 INFO:journalctl@ceph.osd.2.smithi037.stdout:Sep 20 22:25:49 smithi037 podman[76477]: 2024-09-20 22:25:49.217312612 +0000 UTC m=+0.474268686 container init 5f60a1838e85cee3b6fb884222cd5d5f8c87e523f36ab2edb56be3b0f8e36191 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-2-deactivate, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-9e2ac70, GIT_CLEAN=True) 2024-09-20T22:25:49.430 INFO:journalctl@ceph.osd.2.smithi037.stdout:Sep 20 22:25:49 smithi037 podman[76477]: 2024-09-20 22:25:49.232879255 +0000 UTC m=+0.489835317 container start 5f60a1838e85cee3b6fb884222cd5d5f8c87e523f36ab2edb56be3b0f8e36191 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-2-deactivate, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, ceph=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-9e2ac70, org.label-schema.vendor=CentOS) 2024-09-20T22:25:49.430 INFO:journalctl@ceph.osd.2.smithi037.stdout:Sep 20 22:25:49 smithi037 podman[76477]: 2024-09-20 22:25:49.401777324 +0000 UTC m=+0.658733386 container attach 5f60a1838e85cee3b6fb884222cd5d5f8c87e523f36ab2edb56be3b0f8e36191 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-2-deactivate, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-9e2ac70, GIT_CLEAN=True) 2024-09-20T22:25:49.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:49 smithi194 ceph-mon[51725]: pgmap v131: 97 pgs: 2 active+undersized, 5 peering, 15 active+undersized+degraded, 75 active+clean; 334 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.4 MiB/s wr, 684 op/s; 101/1587 objects degraded (6.364%) 2024-09-20T22:25:49.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:49 smithi194 ceph-mon[51725]: Health check failed: Degraded data redundancy: 101/1587 objects degraded (6.364%), 15 pgs degraded (PG_DEGRADED) 2024-09-20T22:25:49.691 INFO:journalctl@ceph.osd.2.smithi037.stdout:Sep 20 22:25:49 smithi037 podman[76477]: 2024-09-20 22:25:49.596451157 +0000 UTC m=+0.853407226 container died 5f60a1838e85cee3b6fb884222cd5d5f8c87e523f36ab2edb56be3b0f8e36191 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-2-deactivate, org.label-schema.license=GPLv2, RELEASE=squid-9e2ac70, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-09-20T22:25:49.858 INFO:teuthology.orchestra.run.smithi037.stdout: 5 201183 10.60 MB/sec execute 367 sec latency 56.580 ms 2024-09-20T22:25:50.726 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:50 smithi037 ceph-mon[65640]: pgmap v132: 97 pgs: 4 active+undersized, 5 peering, 17 active+undersized+degraded, 71 active+clean; 335 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 7.2 MiB/s wr, 704 op/s; 116/1578 objects degraded (7.351%) 2024-09-20T22:25:50.726 INFO:journalctl@ceph.osd.2.smithi037.stdout:Sep 20 22:25:50 smithi037 podman[76499]: 2024-09-20 22:25:50.576245276 +0000 UTC m=+0.970375474 container remove 5f60a1838e85cee3b6fb884222cd5d5f8c87e523f36ab2edb56be3b0f8e36191 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-2-deactivate, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=squid-9e2ac70, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True) 2024-09-20T22:25:50.726 INFO:journalctl@ceph.osd.2.smithi037.stdout:Sep 20 22:25:50 smithi037 systemd[1]: ceph-f9a6357a-779c-11ef-baf5-efdc52797490@osd.2.service: Deactivated successfully. 2024-09-20T22:25:50.727 INFO:journalctl@ceph.osd.2.smithi037.stdout:Sep 20 22:25:50 smithi037 systemd[1]: Stopped Ceph osd.2 for f9a6357a-779c-11ef-baf5-efdc52797490. 2024-09-20T22:25:50.727 INFO:journalctl@ceph.osd.2.smithi037.stdout:Sep 20 22:25:50 smithi037 systemd[1]: ceph-f9a6357a-779c-11ef-baf5-efdc52797490@osd.2.service: Consumed 2min 31.846s CPU time. 2024-09-20T22:25:50.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:50 smithi194 ceph-mon[51725]: pgmap v132: 97 pgs: 4 active+undersized, 5 peering, 17 active+undersized+degraded, 71 active+clean; 335 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 7.2 MiB/s wr, 704 op/s; 116/1578 objects degraded (7.351%) 2024-09-20T22:25:50.858 INFO:teuthology.orchestra.run.smithi037.stdout: 5 201542 10.60 MB/sec execute 368 sec latency 47.342 ms 2024-09-20T22:25:50.976 INFO:journalctl@ceph.osd.2.smithi037.stdout:Sep 20 22:25:50 smithi037 systemd[1]: Starting Ceph osd.2 for f9a6357a-779c-11ef-baf5-efdc52797490... 2024-09-20T22:25:51.290 INFO:journalctl@ceph.osd.2.smithi037.stdout:Sep 20 22:25:51 smithi037 podman[76597]: 2024-09-20 22:25:51.137595066 +0000 UTC m=+0.021387278 image pull cc9251b744a3faafca7c3095f79bfff81873afe823f31a03a11e015a34abcdbd quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5 2024-09-20T22:25:51.291 INFO:journalctl@ceph.osd.2.smithi037.stdout:Sep 20 22:25:51 smithi037 podman[76597]: 2024-09-20 22:25:51.261012738 +0000 UTC m=+0.144804945 container create dbc4d1253673d9516aa521500433baabd0c001192af2fec2b6853926ff1f8a76 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-2-activate, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=squid-9e2ac70, ceph=True, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-09-20T22:25:51.793 INFO:journalctl@ceph.osd.2.smithi037.stdout:Sep 20 22:25:51 smithi037 podman[76597]: 2024-09-20 22:25:51.658353098 +0000 UTC m=+0.542145317 container init dbc4d1253673d9516aa521500433baabd0c001192af2fec2b6853926ff1f8a76 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-2-activate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=squid-9e2ac70, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-09-20T22:25:51.793 INFO:journalctl@ceph.osd.2.smithi037.stdout:Sep 20 22:25:51 smithi037 podman[76597]: 2024-09-20 22:25:51.667678869 +0000 UTC m=+0.551471075 container start dbc4d1253673d9516aa521500433baabd0c001192af2fec2b6853926ff1f8a76 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-2-activate, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=squid-9e2ac70) 2024-09-20T22:25:51.858 INFO:teuthology.orchestra.run.smithi037.stdout: 5 202240 10.63 MB/sec execute 369 sec latency 48.918 ms 2024-09-20T22:25:52.048 INFO:journalctl@ceph.osd.2.smithi037.stdout:Sep 20 22:25:51 smithi037 podman[76597]: 2024-09-20 22:25:51.815720595 +0000 UTC m=+0.699512800 container attach dbc4d1253673d9516aa521500433baabd0c001192af2fec2b6853926ff1f8a76 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-2-activate, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, RELEASE=squid-9e2ac70, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731) 2024-09-20T22:25:52.048 INFO:journalctl@ceph.osd.2.smithi037.stdout:Sep 20 22:25:51 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-2-activate[76610]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T22:25:52.048 INFO:journalctl@ceph.osd.2.smithi037.stdout:Sep 20 22:25:51 smithi037 bash[76597]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T22:25:52.048 INFO:journalctl@ceph.osd.2.smithi037.stdout:Sep 20 22:25:51 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-2-activate[76610]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T22:25:52.048 INFO:journalctl@ceph.osd.2.smithi037.stdout:Sep 20 22:25:51 smithi037 bash[76597]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T22:25:52.858 INFO:teuthology.orchestra.run.smithi037.stdout: 5 202729 10.65 MB/sec execute 370 sec latency 27.464 ms 2024-09-20T22:25:53.057 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:52 smithi037 ceph-mon[65640]: pgmap v133: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 329 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 5.2 MiB/s wr, 451 op/s; 151/1566 objects degraded (9.642%) 2024-09-20T22:25:53.057 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:52 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:25:53.057 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:52 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:25:53.057 INFO:journalctl@ceph.osd.2.smithi037.stdout:Sep 20 22:25:52 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-2-activate[76610]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-20T22:25:53.057 INFO:journalctl@ceph.osd.2.smithi037.stdout:Sep 20 22:25:52 smithi037 bash[76597]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-20T22:25:53.057 INFO:journalctl@ceph.osd.2.smithi037.stdout:Sep 20 22:25:52 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-2-activate[76610]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-20T22:25:53.057 INFO:journalctl@ceph.osd.2.smithi037.stdout:Sep 20 22:25:52 smithi037 bash[76597]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-20T22:25:53.308 INFO:journalctl@ceph.osd.2.smithi037.stdout:Sep 20 22:25:53 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-2-activate[76610]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-20T22:25:53.309 INFO:journalctl@ceph.osd.2.smithi037.stdout:Sep 20 22:25:53 smithi037 bash[76597]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-20T22:25:53.309 INFO:journalctl@ceph.osd.2.smithi037.stdout:Sep 20 22:25:53 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-2-activate[76610]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-20T22:25:53.309 INFO:journalctl@ceph.osd.2.smithi037.stdout:Sep 20 22:25:53 smithi037 bash[76597]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-20T22:25:53.309 INFO:journalctl@ceph.osd.2.smithi037.stdout:Sep 20 22:25:53 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-2-activate[76610]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-09-20T22:25:53.309 INFO:journalctl@ceph.osd.2.smithi037.stdout:Sep 20 22:25:53 smithi037 bash[76597]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-09-20T22:25:53.309 INFO:journalctl@ceph.osd.2.smithi037.stdout:Sep 20 22:25:53 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-2-activate[76610]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-20T22:25:53.309 INFO:journalctl@ceph.osd.2.smithi037.stdout:Sep 20 22:25:53 smithi037 bash[76597]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-20T22:25:53.309 INFO:journalctl@ceph.osd.2.smithi037.stdout:Sep 20 22:25:53 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-2-activate[76610]: --> ceph-volume raw activate successful for osd ID: 2 2024-09-20T22:25:53.309 INFO:journalctl@ceph.osd.2.smithi037.stdout:Sep 20 22:25:53 smithi037 bash[76597]: --> ceph-volume raw activate successful for osd ID: 2 2024-09-20T22:25:53.309 INFO:journalctl@ceph.osd.2.smithi037.stdout:Sep 20 22:25:53 smithi037 conmon[76610]: conmon dbc4d1253673d9516aa5 : Failed to open cgroups file: /sys/fs/cgroup/machine.slice/libpod-dbc4d1253673d9516aa521500433baabd0c001192af2fec2b6853926ff1f8a76.scope/container/memory.events 2024-09-20T22:25:53.309 INFO:journalctl@ceph.osd.2.smithi037.stdout:Sep 20 22:25:53 smithi037 podman[76747]: 2024-09-20 22:25:53.269541199 +0000 UTC m=+0.027211511 container died dbc4d1253673d9516aa521500433baabd0c001192af2fec2b6853926ff1f8a76 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-2-activate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, ceph=True, RELEASE=squid-9e2ac70, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2) 2024-09-20T22:25:53.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:52 smithi194 ceph-mon[51725]: pgmap v133: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 329 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 5.2 MiB/s wr, 451 op/s; 151/1566 objects degraded (9.642%) 2024-09-20T22:25:53.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:52 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:25:53.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:52 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:25:53.858 INFO:teuthology.orchestra.run.smithi037.stdout: 5 203301 10.66 MB/sec execute 371 sec latency 35.209 ms 2024-09-20T22:25:54.060 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:53 smithi037 ceph-mon[65640]: Health check update: Degraded data redundancy: 156/1677 objects degraded (9.302%), 20 pgs degraded (PG_DEGRADED) 2024-09-20T22:25:54.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:53 smithi194 ceph-mon[51725]: Health check update: Degraded data redundancy: 156/1677 objects degraded (9.302%), 20 pgs degraded (PG_DEGRADED) 2024-09-20T22:25:54.562 INFO:journalctl@ceph.osd.2.smithi037.stdout:Sep 20 22:25:54 smithi037 podman[76747]: 2024-09-20 22:25:54.410145962 +0000 UTC m=+1.167816258 container remove dbc4d1253673d9516aa521500433baabd0c001192af2fec2b6853926ff1f8a76 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-2-activate, ceph=True, org.label-schema.schema-version=1.0, RELEASE=squid-9e2ac70, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-20T22:25:54.814 INFO:journalctl@ceph.osd.2.smithi037.stdout:Sep 20 22:25:54 smithi037 podman[76834]: 2024-09-20 22:25:54.57861867 +0000 UTC m=+0.024948867 image pull cc9251b744a3faafca7c3095f79bfff81873afe823f31a03a11e015a34abcdbd quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5 2024-09-20T22:25:54.814 INFO:journalctl@ceph.osd.2.smithi037.stdout:Sep 20 22:25:54 smithi037 podman[76834]: 2024-09-20 22:25:54.780038969 +0000 UTC m=+0.226369162 container create 0922768db2a25e2e82f6de041478e2da194761b05e96027d4eed853dfaf56c76 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-2, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, ceph=True, io.buildah.version=1.37.2, RELEASE=squid-9e2ac70, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-09-20T22:25:54.858 INFO:teuthology.orchestra.run.smithi037.stdout: 5 203689 10.67 MB/sec execute 372 sec latency 50.727 ms 2024-09-20T22:25:55.066 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:54 smithi037 ceph-mon[65640]: pgmap v134: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 337 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.6 KiB/s rd, 8.2 MiB/s wr, 824 op/s; 156/1677 objects degraded (9.302%) 2024-09-20T22:25:55.276 INFO:teuthology.orchestra.run.smithi037.stdout:true 2024-09-20T22:25:55.318 INFO:journalctl@ceph.osd.2.smithi037.stdout:Sep 20 22:25:55 smithi037 podman[76834]: 2024-09-20 22:25:55.232932235 +0000 UTC m=+0.679262428 container init 0922768db2a25e2e82f6de041478e2da194761b05e96027d4eed853dfaf56c76 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-2, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, RELEASE=squid-9e2ac70, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-20T22:25:55.318 INFO:journalctl@ceph.osd.2.smithi037.stdout:Sep 20 22:25:55 smithi037 podman[76834]: 2024-09-20 22:25:55.242935886 +0000 UTC m=+0.689266077 container start 0922768db2a25e2e82f6de041478e2da194761b05e96027d4eed853dfaf56c76 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, RELEASE=squid-9e2ac70, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-20T22:25:55.318 INFO:journalctl@ceph.osd.2.smithi037.stdout:Sep 20 22:25:55 smithi037 ceph-osd[76875]: -- 172.21.15.37:0/110586890 <== mon.0 v2:172.21.15.37:3300/0 3 ==== mon_map magic: 0 ==== 357+0+0 (secure 0 0 0) 0x562c43739ba0 con 0x562c444c0000 2024-09-20T22:25:55.318 INFO:journalctl@ceph.osd.2.smithi037.stdout:Sep 20 22:25:55 smithi037 ceph-osd[76875]: -- 172.21.15.37:0/110586890 <== mon.0 v2:172.21.15.37:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x562c444c2000 con 0x562c444c0000 2024-09-20T22:25:55.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:54 smithi194 ceph-mon[51725]: pgmap v134: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 337 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.6 KiB/s rd, 8.2 MiB/s wr, 824 op/s; 156/1677 objects degraded (9.302%) 2024-09-20T22:25:55.569 INFO:journalctl@ceph.osd.2.smithi037.stdout:Sep 20 22:25:55 smithi037 bash[76834]: 0922768db2a25e2e82f6de041478e2da194761b05e96027d4eed853dfaf56c76 2024-09-20T22:25:55.569 INFO:journalctl@ceph.osd.2.smithi037.stdout:Sep 20 22:25:55 smithi037 systemd[1]: Started Ceph osd.2 for f9a6357a-779c-11ef-baf5-efdc52797490. 2024-09-20T22:25:55.858 INFO:teuthology.orchestra.run.smithi037.stdout: 5 204128 10.66 MB/sec execute 373 sec latency 31.747 ms 2024-09-20T22:25:56.409 INFO:teuthology.orchestra.run.smithi037.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T22:25:56.409 INFO:teuthology.orchestra.run.smithi037.stdout:alertmanager.smithi037 smithi037 *:9093,9094 running (5m) 42s ago 14m 25.8M - 0.25.0 c8568f914cd2 d1f99b3cd707 2024-09-20T22:25:56.409 INFO:teuthology.orchestra.run.smithi037.stdout:crash.smithi037 smithi037 running (2m) 42s ago 14m 6895k - 19.1.1-367-g9e2ac70b cc9251b744a3 77e510d8bdb1 2024-09-20T22:25:56.410 INFO:teuthology.orchestra.run.smithi037.stdout:crash.smithi194 smithi194 running (2m) 2m ago 14m 6895k - 19.1.1-367-g9e2ac70b cc9251b744a3 4d169d1309bc 2024-09-20T22:25:56.410 INFO:teuthology.orchestra.run.smithi037.stdout:grafana.smithi037 smithi037 *:3000 running (5m) 42s ago 14m 54.2M - dad864ee21e9 2306f3c2da19 2024-09-20T22:25:56.410 INFO:teuthology.orchestra.run.smithi037.stdout:mds.foofs.smithi037.sdtedi smithi037 running (8m) 42s ago 8m 38.0M - 17.2.0 e1d6a67b021e 0a7898d25b8d 2024-09-20T22:25:56.410 INFO:teuthology.orchestra.run.smithi037.stdout:mds.foofs.smithi194.lkroso smithi194 running (8m) 2m ago 8m 15.6M - 17.2.0 e1d6a67b021e 17736de30008 2024-09-20T22:25:56.410 INFO:teuthology.orchestra.run.smithi037.stdout:mgr.smithi037.lykblh smithi037 *:8443,9283,8765 running (5m) 42s ago 15m 502M - 19.1.1-367-g9e2ac70b cc9251b744a3 181e46fd9ba6 2024-09-20T22:25:56.410 INFO:teuthology.orchestra.run.smithi037.stdout:mgr.smithi194.bkcglt smithi194 *:8443,9283,8765 running (4m) 2m ago 14m 439M - 19.1.1-367-g9e2ac70b cc9251b744a3 1fa610c97517 2024-09-20T22:25:56.410 INFO:teuthology.orchestra.run.smithi037.stdout:mon.smithi037 smithi037 running (3m) 42s ago 16m 48.5M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 c0229eceb527 2024-09-20T22:25:56.410 INFO:teuthology.orchestra.run.smithi037.stdout:mon.smithi194 smithi194 running (3m) 2m ago 13m 37.2M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 97854807ba91 2024-09-20T22:25:56.410 INFO:teuthology.orchestra.run.smithi037.stdout:nfs.foo.0.0.smithi037.spabdc smithi037 *:2049 running (8m) 42s ago 8m 268M - 4.0 e1d6a67b021e 858ff9d558a2 2024-09-20T22:25:56.410 INFO:teuthology.orchestra.run.smithi037.stdout:nfs.foo.1.0.smithi194.qzjyej smithi194 *:2049 running (8m) 2m ago 8m 89.8M - 4.0 e1d6a67b021e c07fe0eb7c75 2024-09-20T22:25:56.410 INFO:teuthology.orchestra.run.smithi037.stdout:node-exporter.smithi037 smithi037 *:9100 running (5m) 42s ago 14m 16.3M - 1.5.0 0da6a335fe13 433e6aaf6137 2024-09-20T22:25:56.410 INFO:teuthology.orchestra.run.smithi037.stdout:node-exporter.smithi194 smithi194 *:9100 running (5m) 2m ago 13m 15.7M - 1.5.0 0da6a335fe13 1d30bb810ad1 2024-09-20T22:25:56.410 INFO:teuthology.orchestra.run.smithi037.stdout:osd.0 smithi037 running (102s) 42s ago 13m 240M 1937M 19.1.1-367-g9e2ac70b cc9251b744a3 c5568bd43c07 2024-09-20T22:25:56.411 INFO:teuthology.orchestra.run.smithi037.stdout:osd.1 smithi037 running (51s) 42s ago 12m 11.5M 1937M 19.1.1-367-g9e2ac70b cc9251b744a3 f6b715391469 2024-09-20T22:25:56.411 INFO:teuthology.orchestra.run.smithi037.stdout:osd.2 smithi037 starting - - - 1937M 2024-09-20T22:25:56.411 INFO:teuthology.orchestra.run.smithi037.stdout:osd.3 smithi037 running (11m) 42s ago 11m 587M 1937M 17.2.0 e1d6a67b021e 3a22952c6475 2024-09-20T22:25:56.411 INFO:teuthology.orchestra.run.smithi037.stdout:osd.4 smithi194 running (11m) 2m ago 11m 513M 2704M 17.2.0 e1d6a67b021e fa107be8915c 2024-09-20T22:25:56.411 INFO:teuthology.orchestra.run.smithi037.stdout:osd.5 smithi194 running (11m) 2m ago 11m 324M 2704M 17.2.0 e1d6a67b021e 290b21cbd131 2024-09-20T22:25:56.411 INFO:teuthology.orchestra.run.smithi037.stdout:osd.6 smithi194 running (10m) 2m ago 10m 511M 2704M 17.2.0 e1d6a67b021e 27f0720639ed 2024-09-20T22:25:56.411 INFO:teuthology.orchestra.run.smithi037.stdout:osd.7 smithi194 running (10m) 2m ago 10m 406M 2704M 17.2.0 e1d6a67b021e 62d163f4fd92 2024-09-20T22:25:56.411 INFO:teuthology.orchestra.run.smithi037.stdout:prometheus.smithi037 smithi037 *:9095 running (4m) 42s ago 13m 63.8M - 2.43.0 a07b618ecd1d 05b826dde2ff 2024-09-20T22:25:56.583 INFO:journalctl@ceph.osd.2.smithi037.stdout:Sep 20 22:25:56 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-2[76871]: 2024-09-20T22:25:56.424+0000 7f31c4e4d640 -1 Falling back to public interface 2024-09-20T22:25:56.583 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:56 smithi037 ceph-mon[65640]: from='client.34272 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:25:56.583 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:56 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:25:56.583 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:56 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:25:56.583 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:56 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:25:56.583 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:56 smithi037 ceph-mon[65640]: pgmap v135: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 337 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 7.4 MiB/s wr, 702 op/s; 159/1695 objects degraded (9.381%) 2024-09-20T22:25:56.584 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:56 smithi037 ceph-mon[65640]: from='client.34278 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:25:56.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:56 smithi194 ceph-mon[51725]: from='client.34272 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:25:56.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:56 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:25:56.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:56 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:25:56.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:56 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:25:56.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:56 smithi194 ceph-mon[51725]: pgmap v135: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 337 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 7.4 MiB/s wr, 702 op/s; 159/1695 objects degraded (9.381%) 2024-09-20T22:25:56.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:56 smithi194 ceph-mon[51725]: from='client.34278 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:25:56.858 INFO:teuthology.orchestra.run.smithi037.stdout: 5 204656 10.65 MB/sec execute 374 sec latency 39.454 ms 2024-09-20T22:25:57.129 INFO:teuthology.orchestra.run.smithi037.stdout:{ 2024-09-20T22:25:57.129 INFO:teuthology.orchestra.run.smithi037.stdout: "mon": { 2024-09-20T22:25:57.129 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 2 2024-09-20T22:25:57.129 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:25:57.130 INFO:teuthology.orchestra.run.smithi037.stdout: "mgr": { 2024-09-20T22:25:57.130 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 2 2024-09-20T22:25:57.130 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:25:57.130 INFO:teuthology.orchestra.run.smithi037.stdout: "osd": { 2024-09-20T22:25:57.130 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-09-20T22:25:57.130 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 2 2024-09-20T22:25:57.130 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:25:57.130 INFO:teuthology.orchestra.run.smithi037.stdout: "mds": { 2024-09-20T22:25:57.130 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T22:25:57.130 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:25:57.130 INFO:teuthology.orchestra.run.smithi037.stdout: "overall": { 2024-09-20T22:25:57.130 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-09-20T22:25:57.130 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 6 2024-09-20T22:25:57.131 INFO:teuthology.orchestra.run.smithi037.stdout: } 2024-09-20T22:25:57.131 INFO:teuthology.orchestra.run.smithi037.stdout:} 2024-09-20T22:25:57.588 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:57 smithi037 ceph-mon[65640]: from='client.44217 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:25:57.589 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:57 smithi037 ceph-mon[65640]: from='client.? 172.21.15.37:0/3155152572' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:25:57.640 INFO:teuthology.orchestra.run.smithi037.stdout:{ 2024-09-20T22:25:57.641 INFO:teuthology.orchestra.run.smithi037.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5", 2024-09-20T22:25:57.641 INFO:teuthology.orchestra.run.smithi037.stdout: "in_progress": true, 2024-09-20T22:25:57.641 INFO:teuthology.orchestra.run.smithi037.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-20T22:25:57.641 INFO:teuthology.orchestra.run.smithi037.stdout: "services_complete": [ 2024-09-20T22:25:57.641 INFO:teuthology.orchestra.run.smithi037.stdout: "mon", 2024-09-20T22:25:57.641 INFO:teuthology.orchestra.run.smithi037.stdout: "mgr", 2024-09-20T22:25:57.641 INFO:teuthology.orchestra.run.smithi037.stdout: "crash" 2024-09-20T22:25:57.641 INFO:teuthology.orchestra.run.smithi037.stdout: ], 2024-09-20T22:25:57.641 INFO:teuthology.orchestra.run.smithi037.stdout: "progress": "8/23 daemons upgraded", 2024-09-20T22:25:57.641 INFO:teuthology.orchestra.run.smithi037.stdout: "message": "Currently upgrading osd daemons", 2024-09-20T22:25:57.641 INFO:teuthology.orchestra.run.smithi037.stdout: "is_paused": false 2024-09-20T22:25:57.641 INFO:teuthology.orchestra.run.smithi037.stdout:} 2024-09-20T22:25:57.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:57 smithi194 ceph-mon[51725]: from='client.44217 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:25:57.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:57 smithi194 ceph-mon[51725]: from='client.? 172.21.15.37:0/3155152572' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:25:57.858 INFO:teuthology.orchestra.run.smithi037.stdout: 5 205037 10.66 MB/sec execute 375 sec latency 37.348 ms 2024-09-20T22:25:58.185 INFO:teuthology.orchestra.run.smithi037.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 159/1695 objects degraded (9.381%), 20 pgs degraded 2024-09-20T22:25:58.185 INFO:teuthology.orchestra.run.smithi037.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-20T22:25:58.185 INFO:teuthology.orchestra.run.smithi037.stdout: osd.2 (root=default,host=smithi037) is down 2024-09-20T22:25:58.186 INFO:teuthology.orchestra.run.smithi037.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 159/1695 objects degraded (9.381%), 20 pgs degraded 2024-09-20T22:25:58.186 INFO:teuthology.orchestra.run.smithi037.stdout: pg 2.1 is active+undersized+degraded, acting [3,0] 2024-09-20T22:25:58.186 INFO:teuthology.orchestra.run.smithi037.stdout: pg 2.3 is active+undersized+degraded, acting [5,7] 2024-09-20T22:25:58.186 INFO:teuthology.orchestra.run.smithi037.stdout: pg 2.7 is active+undersized+degraded, acting [6,7] 2024-09-20T22:25:58.186 INFO:teuthology.orchestra.run.smithi037.stdout: pg 2.c is active+undersized+degraded, acting [5,0] 2024-09-20T22:25:58.186 INFO:teuthology.orchestra.run.smithi037.stdout: pg 2.e is active+undersized+degraded, acting [3,7] 2024-09-20T22:25:58.186 INFO:teuthology.orchestra.run.smithi037.stdout: pg 2.10 is active+undersized+degraded, acting [1,0] 2024-09-20T22:25:58.186 INFO:teuthology.orchestra.run.smithi037.stdout: pg 2.16 is active+undersized+degraded, acting [5,6] 2024-09-20T22:25:58.186 INFO:teuthology.orchestra.run.smithi037.stdout: pg 2.17 is active+undersized+degraded, acting [6,5] 2024-09-20T22:25:58.186 INFO:teuthology.orchestra.run.smithi037.stdout: pg 2.1c is active+undersized+degraded, acting [7,5] 2024-09-20T22:25:58.186 INFO:teuthology.orchestra.run.smithi037.stdout: pg 2.1f is active+undersized+degraded, acting [0,4] 2024-09-20T22:25:58.186 INFO:teuthology.orchestra.run.smithi037.stdout: pg 3.0 is active+undersized+degraded, acting [1,6] 2024-09-20T22:25:58.186 INFO:teuthology.orchestra.run.smithi037.stdout: pg 3.4 is active+undersized+degraded, acting [1,5] 2024-09-20T22:25:58.186 INFO:teuthology.orchestra.run.smithi037.stdout: pg 3.5 is active+undersized+degraded, acting [5,3] 2024-09-20T22:25:58.187 INFO:teuthology.orchestra.run.smithi037.stdout: pg 3.9 is active+undersized+degraded, acting [4,7] 2024-09-20T22:25:58.187 INFO:teuthology.orchestra.run.smithi037.stdout: pg 3.13 is active+undersized+degraded, acting [7,4] 2024-09-20T22:25:58.187 INFO:teuthology.orchestra.run.smithi037.stdout: pg 3.1a is active+undersized+degraded, acting [4,1] 2024-09-20T22:25:58.187 INFO:teuthology.orchestra.run.smithi037.stdout: pg 3.1e is active+undersized+degraded, acting [3,6] 2024-09-20T22:25:58.187 INFO:teuthology.orchestra.run.smithi037.stdout: pg 3.1f is active+undersized+degraded, acting [0,5] 2024-09-20T22:25:58.187 INFO:teuthology.orchestra.run.smithi037.stdout: pg 4.14 is active+undersized+degraded, acting [3,7] 2024-09-20T22:25:58.187 INFO:teuthology.orchestra.run.smithi037.stdout: pg 4.19 is active+undersized+degraded, acting [3,0] 2024-09-20T22:25:58.599 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:58 smithi037 ceph-mon[65640]: from='client.34290 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:25:58.600 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:58 smithi037 ceph-mon[65640]: pgmap v136: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 334 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 9.2 MiB/s wr, 936 op/s; 143/1575 objects degraded (9.079%) 2024-09-20T22:25:58.600 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:58 smithi037 ceph-mon[65640]: from='client.? 172.21.15.37:0/2884194331' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T22:25:58.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:58 smithi194 ceph-mon[51725]: from='client.34290 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:25:58.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:58 smithi194 ceph-mon[51725]: pgmap v136: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 334 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 9.2 MiB/s wr, 936 op/s; 143/1575 objects degraded (9.079%) 2024-09-20T22:25:58.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:58 smithi194 ceph-mon[51725]: from='client.? 172.21.15.37:0/2884194331' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T22:25:58.858 INFO:teuthology.orchestra.run.smithi037.stdout: 5 205405 10.67 MB/sec execute 376 sec latency 85.311 ms 2024-09-20T22:25:59.651 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:25:59 smithi037 ceph-mon[65640]: Health check update: Degraded data redundancy: 143/1575 objects degraded (9.079%), 20 pgs degraded (PG_DEGRADED) 2024-09-20T22:25:59.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:25:59 smithi194 ceph-mon[51725]: Health check update: Degraded data redundancy: 143/1575 objects degraded (9.079%), 20 pgs degraded (PG_DEGRADED) 2024-09-20T22:25:59.858 INFO:teuthology.orchestra.run.smithi037.stdout: 5 205761 10.66 MB/sec execute 377 sec latency 60.891 ms 2024-09-20T22:26:00.732 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:00 smithi037 ceph-mon[65640]: pgmap v137: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 334 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 6.2 MiB/s wr, 603 op/s; 140/1551 objects degraded (9.026%) 2024-09-20T22:26:00.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:00 smithi194 ceph-mon[51725]: pgmap v137: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 334 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 6.2 MiB/s wr, 603 op/s; 140/1551 objects degraded (9.026%) 2024-09-20T22:26:00.858 INFO:teuthology.orchestra.run.smithi037.stdout: 5 205982 10.65 MB/sec execute 378 sec latency 82.992 ms 2024-09-20T22:26:01.858 INFO:teuthology.orchestra.run.smithi037.stdout: 5 206177 10.64 MB/sec execute 379 sec latency 67.755 ms 2024-09-20T22:26:02.859 INFO:teuthology.orchestra.run.smithi037.stdout: 5 206433 10.64 MB/sec execute 380 sec latency 54.515 ms 2024-09-20T22:26:03.081 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:02 smithi037 ceph-mon[65640]: pgmap v138: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 331 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.8 MiB/s wr, 590 op/s; 143/1557 objects degraded (9.184%) 2024-09-20T22:26:03.335 INFO:journalctl@ceph.osd.2.smithi037.stdout:Sep 20 22:26:03 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-2[76871]: 2024-09-20T22:26:03.321+0000 7f31c4e4d640 -1 osd.2 0 read_superblock omap replica is missing. 2024-09-20T22:26:03.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:02 smithi194 ceph-mon[51725]: pgmap v138: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 331 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.8 MiB/s wr, 590 op/s; 143/1557 objects degraded (9.184%) 2024-09-20T22:26:03.859 INFO:teuthology.orchestra.run.smithi037.stdout: 5 206687 10.62 MB/sec execute 381 sec latency 54.396 ms 2024-09-20T22:26:04.352 INFO:journalctl@ceph.osd.2.smithi037.stdout:Sep 20 22:26:04 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-2[76871]: 2024-09-20T22:26:04.294+0000 7f31c4e4d640 -1 osd.2 96 log_to_monitors true 2024-09-20T22:26:04.859 INFO:teuthology.orchestra.run.smithi037.stdout: 5 206932 10.61 MB/sec execute 382 sec latency 52.489 ms 2024-09-20T22:26:04.863 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:04 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:26:04.863 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:04 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:26:04.863 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:04 smithi037 ceph-mon[65640]: pgmap v139: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 338 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.9 MiB/s wr, 711 op/s; 153/1620 objects degraded (9.444%) 2024-09-20T22:26:04.863 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:04 smithi037 ceph-mon[65640]: from='osd.2 [v2:172.21.15.37:6818/4066027029,v1:172.21.15.37:6819/4066027029]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-20T22:26:04.863 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:04 smithi037 ceph-mon[65640]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-20T22:26:04.863 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:04 smithi037 ceph-mon[65640]: Health check update: Degraded data redundancy: 143/1557 objects degraded (9.184%), 20 pgs degraded (PG_DEGRADED) 2024-09-20T22:26:04.863 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:04 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:26:04.863 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:04 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:26:05.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:04 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:26:05.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:04 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:26:05.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:04 smithi194 ceph-mon[51725]: pgmap v139: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 338 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.9 MiB/s wr, 711 op/s; 153/1620 objects degraded (9.444%) 2024-09-20T22:26:05.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:04 smithi194 ceph-mon[51725]: from='osd.2 [v2:172.21.15.37:6818/4066027029,v1:172.21.15.37:6819/4066027029]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-20T22:26:05.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:04 smithi194 ceph-mon[51725]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-20T22:26:05.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:04 smithi194 ceph-mon[51725]: Health check update: Degraded data redundancy: 143/1557 objects degraded (9.184%), 20 pgs degraded (PG_DEGRADED) 2024-09-20T22:26:05.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:04 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:26:05.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:04 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:26:05.666 INFO:journalctl@ceph.osd.2.smithi037.stdout:Sep 20 22:26:05 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-2[76871]: 2024-09-20T22:26:05.416+0000 7f31b8bf0640 -1 osd.2 96 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-20T22:26:05.859 INFO:teuthology.orchestra.run.smithi037.stdout: 5 207106 10.60 MB/sec execute 383 sec latency 169.181 ms 2024-09-20T22:26:05.916 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:05 smithi037 ceph-mon[65640]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-20T22:26:05.917 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:05 smithi037 ceph-mon[65640]: from='osd.2 [v2:172.21.15.37:6818/4066027029,v1:172.21.15.37:6819/4066027029]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi037", "root=default"]}]: dispatch 2024-09-20T22:26:05.917 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:05 smithi037 ceph-mon[65640]: osdmap e99: 8 total, 7 up, 8 in 2024-09-20T22:26:05.917 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:05 smithi037 ceph-mon[65640]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi037", "root=default"]}]: dispatch 2024-09-20T22:26:05.917 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:05 smithi037 ceph-mon[65640]: from='osd.2 ' entity='osd.2' 2024-09-20T22:26:06.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:05 smithi194 ceph-mon[51725]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-20T22:26:06.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:05 smithi194 ceph-mon[51725]: from='osd.2 [v2:172.21.15.37:6818/4066027029,v1:172.21.15.37:6819/4066027029]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi037", "root=default"]}]: dispatch 2024-09-20T22:26:06.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:05 smithi194 ceph-mon[51725]: osdmap e99: 8 total, 7 up, 8 in 2024-09-20T22:26:06.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:05 smithi194 ceph-mon[51725]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi037", "root=default"]}]: dispatch 2024-09-20T22:26:06.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:05 smithi194 ceph-mon[51725]: from='osd.2 ' entity='osd.2' 2024-09-20T22:26:06.859 INFO:teuthology.orchestra.run.smithi037.stdout: 5 207409 10.60 MB/sec execute 384 sec latency 264.955 ms 2024-09-20T22:26:06.932 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:06 smithi037 ceph-mon[65640]: pgmap v141: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 339 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.4 MiB/s wr, 528 op/s; 154/1635 objects degraded (9.419%) 2024-09-20T22:26:06.933 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:06 smithi037 ceph-mon[65640]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-20T22:26:06.933 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:06 smithi037 ceph-mon[65640]: osd.2 [v2:172.21.15.37:6818/4066027029,v1:172.21.15.37:6819/4066027029] boot 2024-09-20T22:26:06.933 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:06 smithi037 ceph-mon[65640]: osdmap e100: 8 total, 8 up, 8 in 2024-09-20T22:26:06.933 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:06 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-20T22:26:07.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:06 smithi194 ceph-mon[51725]: pgmap v141: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 339 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.4 MiB/s wr, 528 op/s; 154/1635 objects degraded (9.419%) 2024-09-20T22:26:07.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:06 smithi194 ceph-mon[51725]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-20T22:26:07.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:06 smithi194 ceph-mon[51725]: osd.2 [v2:172.21.15.37:6818/4066027029,v1:172.21.15.37:6819/4066027029] boot 2024-09-20T22:26:07.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:06 smithi194 ceph-mon[51725]: osdmap e100: 8 total, 8 up, 8 in 2024-09-20T22:26:07.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:06 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-20T22:26:07.859 INFO:teuthology.orchestra.run.smithi037.stdout: 5 207557 10.57 MB/sec execute 385 sec latency 1264.989 ms 2024-09-20T22:26:08.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:07 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:26:08.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:07 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:26:08.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:07 smithi037 ceph-mon[65640]: osdmap e101: 8 total, 8 up, 8 in 2024-09-20T22:26:08.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:07 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:26:08.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:07 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:26:08.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:07 smithi194 ceph-mon[51725]: osdmap e101: 8 total, 8 up, 8 in 2024-09-20T22:26:08.860 INFO:teuthology.orchestra.run.smithi037.stdout: 5 207653 10.55 MB/sec execute 386 sec latency 1857.856 ms 2024-09-20T22:26:08.958 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:08 smithi037 ceph-mon[65640]: pgmap v144: 97 pgs: 6 remapped+peering, 5 active+undersized, 8 active+undersized+degraded, 78 active+clean; 338 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.0 KiB/s rd, 5.8 MiB/s wr, 576 op/s; 63/1569 objects degraded (4.015%); 0 B/s, 0 objects/s recovering 2024-09-20T22:26:08.958 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:08 smithi037 ceph-mon[65640]: osdmap e102: 8 total, 8 up, 8 in 2024-09-20T22:26:09.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:08 smithi194 ceph-mon[51725]: pgmap v144: 97 pgs: 6 remapped+peering, 5 active+undersized, 8 active+undersized+degraded, 78 active+clean; 338 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.0 KiB/s rd, 5.8 MiB/s wr, 576 op/s; 63/1569 objects degraded (4.015%); 0 B/s, 0 objects/s recovering 2024-09-20T22:26:09.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:08 smithi194 ceph-mon[51725]: osdmap e102: 8 total, 8 up, 8 in 2024-09-20T22:26:09.860 INFO:teuthology.orchestra.run.smithi037.stdout: 5 208010 10.54 MB/sec execute 387 sec latency 477.573 ms 2024-09-20T22:26:10.642 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:10 smithi037 ceph-mon[65640]: osdmap e103: 8 total, 8 up, 8 in 2024-09-20T22:26:10.642 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:10 smithi037 ceph-mon[65640]: Health check update: Degraded data redundancy: 63/1569 objects degraded (4.015%), 8 pgs degraded (PG_DEGRADED) 2024-09-20T22:26:10.642 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:10 smithi037 ceph-mon[65640]: pgmap v147: 97 pgs: 1 peering, 6 remapped+peering, 3 active+undersized, 6 active+undersized+degraded, 81 active+clean; 335 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.2 MiB/s wr, 467 op/s; 47/1533 objects degraded (3.066%); 162 KiB/s, 4 objects/s recovering 2024-09-20T22:26:10.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:10 smithi194 ceph-mon[51725]: osdmap e103: 8 total, 8 up, 8 in 2024-09-20T22:26:10.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:10 smithi194 ceph-mon[51725]: Health check update: Degraded data redundancy: 63/1569 objects degraded (4.015%), 8 pgs degraded (PG_DEGRADED) 2024-09-20T22:26:10.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:10 smithi194 ceph-mon[51725]: pgmap v147: 97 pgs: 1 peering, 6 remapped+peering, 3 active+undersized, 6 active+undersized+degraded, 81 active+clean; 335 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.2 MiB/s wr, 467 op/s; 47/1533 objects degraded (3.066%); 162 KiB/s, 4 objects/s recovering 2024-09-20T22:26:10.860 INFO:teuthology.orchestra.run.smithi037.stdout: 5 208207 10.52 MB/sec execute 388 sec latency 1020.906 ms 2024-09-20T22:26:11.409 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:11 smithi037 ceph-mon[65640]: osdmap e104: 8 total, 8 up, 8 in 2024-09-20T22:26:11.409 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:11 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:26:11.409 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:11 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:26:11.409 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:11 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:26:11.409 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:11 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:26:11.409 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:11 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:26:11.409 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:11 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:26:11.409 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:11 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:26:11.409 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:11 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:26:11.410 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:11 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:26:11.410 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:11 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:26:11.410 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:11 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-20T22:26:11.410 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:11 smithi037 ceph-mon[65640]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-20T22:26:11.410 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:11 smithi037 ceph-mon[65640]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-09-20T22:26:11.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:11 smithi194 ceph-mon[51725]: osdmap e104: 8 total, 8 up, 8 in 2024-09-20T22:26:11.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:11 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:26:11.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:11 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:26:11.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:11 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:26:11.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:11 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:26:11.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:11 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:26:11.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:11 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:26:11.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:11 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:26:11.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:11 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:26:11.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:11 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:26:11.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:11 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:26:11.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:11 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-20T22:26:11.846 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:11 smithi194 ceph-mon[51725]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-20T22:26:11.846 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:11 smithi194 ceph-mon[51725]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-09-20T22:26:11.860 INFO:teuthology.orchestra.run.smithi037.stdout: 5 208523 10.50 MB/sec execute 389 sec latency 996.242 ms 2024-09-20T22:26:12.420 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:12 smithi037 ceph-mon[65640]: osdmap e105: 8 total, 8 up, 8 in 2024-09-20T22:26:12.420 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:12 smithi037 ceph-mon[65640]: pgmap v150: 97 pgs: 2 peering, 6 remapped+peering, 89 active+clean; 335 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 650 KiB/s wr, 50 op/s; 267 KiB/s, 18 objects/s recovering 2024-09-20T22:26:12.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:12 smithi194 ceph-mon[51725]: osdmap e105: 8 total, 8 up, 8 in 2024-09-20T22:26:12.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:12 smithi194 ceph-mon[51725]: pgmap v150: 97 pgs: 2 peering, 6 remapped+peering, 89 active+clean; 335 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 650 KiB/s wr, 50 op/s; 267 KiB/s, 18 objects/s recovering 2024-09-20T22:26:12.860 INFO:teuthology.orchestra.run.smithi037.stdout: 5 208873 10.51 MB/sec execute 390 sec latency 39.238 ms 2024-09-20T22:26:13.442 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:13 smithi037 ceph-mon[65640]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 47/1533 objects degraded (3.066%), 6 pgs degraded) 2024-09-20T22:26:13.442 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:13 smithi037 ceph-mon[65640]: Cluster is now healthy 2024-09-20T22:26:13.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:13 smithi194 ceph-mon[51725]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 47/1533 objects degraded (3.066%), 6 pgs degraded) 2024-09-20T22:26:13.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:13 smithi194 ceph-mon[51725]: Cluster is now healthy 2024-09-20T22:26:13.860 INFO:teuthology.orchestra.run.smithi037.stdout: 5 209388 10.53 MB/sec execute 391 sec latency 45.565 ms 2024-09-20T22:26:14.477 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:14 smithi037 ceph-mon[65640]: pgmap v151: 97 pgs: 2 peering, 95 active+clean; 334 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 2.8 MiB/s wr, 354 op/s; 1.6 MiB/s, 40 objects/s recovering 2024-09-20T22:26:14.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:14 smithi194 ceph-mon[51725]: pgmap v151: 97 pgs: 2 peering, 95 active+clean; 334 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 2.8 MiB/s wr, 354 op/s; 1.6 MiB/s, 40 objects/s recovering 2024-09-20T22:26:14.860 INFO:teuthology.orchestra.run.smithi037.stdout: 5 209827 10.55 MB/sec execute 392 sec latency 33.371 ms 2024-09-20T22:26:15.860 INFO:teuthology.orchestra.run.smithi037.stdout: 5 210170 10.55 MB/sec execute 393 sec latency 30.115 ms 2024-09-20T22:26:16.860 INFO:teuthology.orchestra.run.smithi037.stdout: 5 210595 10.54 MB/sec execute 394 sec latency 35.653 ms 2024-09-20T22:26:17.052 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:16 smithi037 ceph-mon[65640]: pgmap v152: 97 pgs: 1 peering, 96 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.4 KiB/s rd, 3.2 MiB/s wr, 337 op/s; 1.4 MiB/s, 34 objects/s recovering 2024-09-20T22:26:17.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:16 smithi194 ceph-mon[51725]: pgmap v152: 97 pgs: 1 peering, 96 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.4 KiB/s rd, 3.2 MiB/s wr, 337 op/s; 1.4 MiB/s, 34 objects/s recovering 2024-09-20T22:26:17.860 INFO:teuthology.orchestra.run.smithi037.stdout: 5 210914 10.53 MB/sec execute 395 sec latency 43.321 ms 2024-09-20T22:26:18.860 INFO:teuthology.orchestra.run.smithi037.stdout: 5 211389 10.54 MB/sec execute 396 sec latency 41.303 ms 2024-09-20T22:26:19.088 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:18 smithi037 ceph-mon[65640]: pgmap v153: 97 pgs: 97 active+clean; 350 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.8 MiB/s wr, 594 op/s; 1.1 MiB/s, 25 objects/s recovering 2024-09-20T22:26:19.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:18 smithi194 ceph-mon[51725]: pgmap v153: 97 pgs: 97 active+clean; 350 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.8 MiB/s wr, 594 op/s; 1.1 MiB/s, 25 objects/s recovering 2024-09-20T22:26:19.860 INFO:teuthology.orchestra.run.smithi037.stdout: 5 211901 10.52 MB/sec execute 397 sec latency 51.240 ms 2024-09-20T22:26:20.860 INFO:teuthology.orchestra.run.smithi037.stdout: 5 212274 10.52 MB/sec execute 398 sec latency 45.573 ms 2024-09-20T22:26:21.141 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:20 smithi037 ceph-mon[65640]: pgmap v154: 97 pgs: 97 active+clean; 349 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 2.3 KiB/s rd, 5.4 MiB/s wr, 523 op/s; 893 KiB/s, 21 objects/s recovering 2024-09-20T22:26:21.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:20 smithi194 ceph-mon[51725]: pgmap v154: 97 pgs: 97 active+clean; 349 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 2.3 KiB/s rd, 5.4 MiB/s wr, 523 op/s; 893 KiB/s, 21 objects/s recovering 2024-09-20T22:26:21.860 INFO:teuthology.orchestra.run.smithi037.stdout: 5 212754 10.54 MB/sec execute 399 sec latency 48.137 ms 2024-09-20T22:26:22.860 INFO:teuthology.orchestra.run.smithi037.stdout: 5 213173 10.55 MB/sec execute 400 sec latency 40.297 ms 2024-09-20T22:26:23.018 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:22 smithi037 ceph-mon[65640]: pgmap v155: 97 pgs: 97 active+clean; 346 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.3 MiB/s wr, 547 op/s; 782 KiB/s, 14 objects/s recovering 2024-09-20T22:26:23.019 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:22 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:26:23.019 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:22 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:26:23.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:22 smithi194 ceph-mon[51725]: pgmap v155: 97 pgs: 97 active+clean; 346 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.3 MiB/s wr, 547 op/s; 782 KiB/s, 14 objects/s recovering 2024-09-20T22:26:23.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:22 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:26:23.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:22 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:26:23.860 INFO:teuthology.orchestra.run.smithi037.stdout: 5 213555 10.57 MB/sec execute 401 sec latency 111.861 ms 2024-09-20T22:26:24.860 INFO:teuthology.orchestra.run.smithi037.stdout: 5 213786 10.55 MB/sec execute 402 sec latency 64.644 ms 2024-09-20T22:26:25.050 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:24 smithi037 ceph-mon[65640]: pgmap v156: 97 pgs: 97 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 6.2 MiB/s wr, 604 op/s; 682 KiB/s, 12 objects/s recovering 2024-09-20T22:26:25.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:24 smithi194 ceph-mon[51725]: pgmap v156: 97 pgs: 97 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 6.2 MiB/s wr, 604 op/s; 682 KiB/s, 12 objects/s recovering 2024-09-20T22:26:25.860 INFO:teuthology.orchestra.run.smithi037.stdout: 5 214073 10.54 MB/sec execute 403 sec latency 53.369 ms 2024-09-20T22:26:26.073 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:25 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-20T22:26:26.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:25 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-20T22:26:26.861 INFO:teuthology.orchestra.run.smithi037.stdout: 5 214300 10.52 MB/sec execute 404 sec latency 54.337 ms 2024-09-20T22:26:27.106 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:26 smithi037 ceph-mon[65640]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-20T22:26:27.106 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:26 smithi037 ceph-mon[65640]: Upgrade: osd.3 is safe to restart 2024-09-20T22:26:27.106 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:26 smithi037 ceph-mon[65640]: pgmap v157: 97 pgs: 97 active+clean; 337 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.6 MiB/s wr, 476 op/s 2024-09-20T22:26:27.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:26 smithi194 ceph-mon[51725]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-20T22:26:27.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:26 smithi194 ceph-mon[51725]: Upgrade: osd.3 is safe to restart 2024-09-20T22:26:27.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:26 smithi194 ceph-mon[51725]: pgmap v157: 97 pgs: 97 active+clean; 337 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.6 MiB/s wr, 476 op/s 2024-09-20T22:26:27.860 INFO:teuthology.orchestra.run.smithi037.stdout: 5 214570 10.52 MB/sec execute 405 sec latency 61.463 ms 2024-09-20T22:26:28.118 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:27 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:26:28.245 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:27 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:26:28.729 INFO:teuthology.orchestra.run.smithi037.stdout:true 2024-09-20T22:26:28.860 INFO:teuthology.orchestra.run.smithi037.stdout: 5 214887 10.51 MB/sec execute 406 sec latency 62.853 ms 2024-09-20T22:26:29.128 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:28 smithi037 ceph-mon[65640]: pgmap v158: 97 pgs: 97 active+clean; 342 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 7.0 MiB/s wr, 680 op/s 2024-09-20T22:26:29.128 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:28 smithi037 ceph-mon[65640]: Upgrade: Updating osd.3 2024-09-20T22:26:29.128 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:28 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:26:29.128 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:28 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-20T22:26:29.128 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:28 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:26:29.128 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:28 smithi037 ceph-mon[65640]: Deploying daemon osd.3 on smithi037 2024-09-20T22:26:29.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:28 smithi194 ceph-mon[51725]: pgmap v158: 97 pgs: 97 active+clean; 342 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 7.0 MiB/s wr, 680 op/s 2024-09-20T22:26:29.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:28 smithi194 ceph-mon[51725]: Upgrade: Updating osd.3 2024-09-20T22:26:29.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:28 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:26:29.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:28 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-20T22:26:29.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:28 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:26:29.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:28 smithi194 ceph-mon[51725]: Deploying daemon osd.3 on smithi037 2024-09-20T22:26:29.860 INFO:teuthology.orchestra.run.smithi037.stdout: 5 215499 10.50 MB/sec execute 407 sec latency 81.655 ms 2024-09-20T22:26:29.862 INFO:teuthology.orchestra.run.smithi037.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T22:26:29.862 INFO:teuthology.orchestra.run.smithi037.stdout:alertmanager.smithi037 smithi037 *:9093,9094 running (6m) 26s ago 15m 25.8M - 0.25.0 c8568f914cd2 d1f99b3cd707 2024-09-20T22:26:29.862 INFO:teuthology.orchestra.run.smithi037.stdout:crash.smithi037 smithi037 running (2m) 26s ago 15m 6895k - 19.1.1-367-g9e2ac70b cc9251b744a3 77e510d8bdb1 2024-09-20T22:26:29.862 INFO:teuthology.orchestra.run.smithi037.stdout:crash.smithi194 smithi194 running (2m) 2m ago 14m 6895k - 19.1.1-367-g9e2ac70b cc9251b744a3 4d169d1309bc 2024-09-20T22:26:29.862 INFO:teuthology.orchestra.run.smithi037.stdout:grafana.smithi037 smithi037 *:3000 running (6m) 26s ago 15m 52.8M - dad864ee21e9 2306f3c2da19 2024-09-20T22:26:29.862 INFO:teuthology.orchestra.run.smithi037.stdout:mds.foofs.smithi037.sdtedi smithi037 running (9m) 26s ago 9m 37.8M - 17.2.0 e1d6a67b021e 0a7898d25b8d 2024-09-20T22:26:29.862 INFO:teuthology.orchestra.run.smithi037.stdout:mds.foofs.smithi194.lkroso smithi194 running (9m) 2m ago 9m 15.6M - 17.2.0 e1d6a67b021e 17736de30008 2024-09-20T22:26:29.862 INFO:teuthology.orchestra.run.smithi037.stdout:mgr.smithi037.lykblh smithi037 *:8443,9283,8765 running (5m) 26s ago 16m 504M - 19.1.1-367-g9e2ac70b cc9251b744a3 181e46fd9ba6 2024-09-20T22:26:29.862 INFO:teuthology.orchestra.run.smithi037.stdout:mgr.smithi194.bkcglt smithi194 *:8443,9283,8765 running (4m) 2m ago 14m 439M - 19.1.1-367-g9e2ac70b cc9251b744a3 1fa610c97517 2024-09-20T22:26:29.863 INFO:teuthology.orchestra.run.smithi037.stdout:mon.smithi037 smithi037 running (4m) 26s ago 16m 50.9M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 c0229eceb527 2024-09-20T22:26:29.863 INFO:teuthology.orchestra.run.smithi037.stdout:mon.smithi194 smithi194 running (3m) 2m ago 14m 37.2M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 97854807ba91 2024-09-20T22:26:29.863 INFO:teuthology.orchestra.run.smithi037.stdout:nfs.foo.0.0.smithi037.spabdc smithi037 *:2049 running (9m) 26s ago 8m 281M - 4.0 e1d6a67b021e 858ff9d558a2 2024-09-20T22:26:29.863 INFO:teuthology.orchestra.run.smithi037.stdout:nfs.foo.1.0.smithi194.qzjyej smithi194 *:2049 running (8m) 2m ago 8m 89.8M - 4.0 e1d6a67b021e c07fe0eb7c75 2024-09-20T22:26:29.863 INFO:teuthology.orchestra.run.smithi037.stdout:node-exporter.smithi037 smithi037 *:9100 running (6m) 26s ago 14m 15.4M - 1.5.0 0da6a335fe13 433e6aaf6137 2024-09-20T22:26:29.863 INFO:teuthology.orchestra.run.smithi037.stdout:node-exporter.smithi194 smithi194 *:9100 running (5m) 2m ago 14m 15.7M - 1.5.0 0da6a335fe13 1d30bb810ad1 2024-09-20T22:26:29.863 INFO:teuthology.orchestra.run.smithi037.stdout:osd.0 smithi037 running (2m) 26s ago 13m 309M 1937M 19.1.1-367-g9e2ac70b cc9251b744a3 c5568bd43c07 2024-09-20T22:26:29.863 INFO:teuthology.orchestra.run.smithi037.stdout:osd.1 smithi037 running (85s) 26s ago 13m 226M 1937M 19.1.1-367-g9e2ac70b cc9251b744a3 f6b715391469 2024-09-20T22:26:29.863 INFO:teuthology.orchestra.run.smithi037.stdout:osd.2 smithi037 running (35s) 26s ago 12m 11.2M 1937M 19.1.1-367-g9e2ac70b cc9251b744a3 0922768db2a2 2024-09-20T22:26:29.863 INFO:teuthology.orchestra.run.smithi037.stdout:osd.3 smithi037 running (12m) 26s ago 12m 377M 1937M 17.2.0 e1d6a67b021e 3a22952c6475 2024-09-20T22:26:29.863 INFO:teuthology.orchestra.run.smithi037.stdout:osd.4 smithi194 running (11m) 2m ago 11m 513M 2704M 17.2.0 e1d6a67b021e fa107be8915c 2024-09-20T22:26:29.863 INFO:teuthology.orchestra.run.smithi037.stdout:osd.5 smithi194 running (11m) 2m ago 11m 324M 2704M 17.2.0 e1d6a67b021e 290b21cbd131 2024-09-20T22:26:29.863 INFO:teuthology.orchestra.run.smithi037.stdout:osd.6 smithi194 running (11m) 2m ago 11m 511M 2704M 17.2.0 e1d6a67b021e 27f0720639ed 2024-09-20T22:26:29.864 INFO:teuthology.orchestra.run.smithi037.stdout:osd.7 smithi194 running (10m) 2m ago 10m 406M 2704M 17.2.0 e1d6a67b021e 62d163f4fd92 2024-09-20T22:26:29.864 INFO:teuthology.orchestra.run.smithi037.stdout:prometheus.smithi037 smithi037 *:9095 running (4m) 26s ago 14m 63.8M - 2.43.0 a07b618ecd1d 05b826dde2ff 2024-09-20T22:26:30.142 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:29 smithi037 ceph-mon[65640]: from='client.44233 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:26:30.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:29 smithi194 ceph-mon[51725]: from='client.44233 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:26:30.436 INFO:teuthology.orchestra.run.smithi037.stdout:{ 2024-09-20T22:26:30.437 INFO:teuthology.orchestra.run.smithi037.stdout: "mon": { 2024-09-20T22:26:30.437 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 2 2024-09-20T22:26:30.437 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:26:30.437 INFO:teuthology.orchestra.run.smithi037.stdout: "mgr": { 2024-09-20T22:26:30.437 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 2 2024-09-20T22:26:30.437 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:26:30.437 INFO:teuthology.orchestra.run.smithi037.stdout: "osd": { 2024-09-20T22:26:30.437 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-09-20T22:26:30.437 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 3 2024-09-20T22:26:30.437 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:26:30.437 INFO:teuthology.orchestra.run.smithi037.stdout: "mds": { 2024-09-20T22:26:30.437 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T22:26:30.437 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:26:30.438 INFO:teuthology.orchestra.run.smithi037.stdout: "overall": { 2024-09-20T22:26:30.438 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-09-20T22:26:30.438 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 7 2024-09-20T22:26:30.438 INFO:teuthology.orchestra.run.smithi037.stdout: } 2024-09-20T22:26:30.438 INFO:teuthology.orchestra.run.smithi037.stdout:} 2024-09-20T22:26:30.861 INFO:teuthology.orchestra.run.smithi037.stdout: 5 215825 10.50 MB/sec execute 408 sec latency 61.503 ms 2024-09-20T22:26:30.984 INFO:teuthology.orchestra.run.smithi037.stdout:{ 2024-09-20T22:26:30.984 INFO:teuthology.orchestra.run.smithi037.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5", 2024-09-20T22:26:30.984 INFO:teuthology.orchestra.run.smithi037.stdout: "in_progress": true, 2024-09-20T22:26:30.984 INFO:teuthology.orchestra.run.smithi037.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-20T22:26:30.984 INFO:teuthology.orchestra.run.smithi037.stdout: "services_complete": [ 2024-09-20T22:26:30.984 INFO:teuthology.orchestra.run.smithi037.stdout: "mon", 2024-09-20T22:26:30.984 INFO:teuthology.orchestra.run.smithi037.stdout: "mgr", 2024-09-20T22:26:30.984 INFO:teuthology.orchestra.run.smithi037.stdout: "crash" 2024-09-20T22:26:30.984 INFO:teuthology.orchestra.run.smithi037.stdout: ], 2024-09-20T22:26:30.984 INFO:teuthology.orchestra.run.smithi037.stdout: "progress": "9/23 daemons upgraded", 2024-09-20T22:26:30.984 INFO:teuthology.orchestra.run.smithi037.stdout: "message": "Currently upgrading osd daemons", 2024-09-20T22:26:30.985 INFO:teuthology.orchestra.run.smithi037.stdout: "is_paused": false 2024-09-20T22:26:30.985 INFO:teuthology.orchestra.run.smithi037.stdout:} 2024-09-20T22:26:31.154 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:30 smithi037 ceph-mon[65640]: from='client.34302 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:26:31.154 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:30 smithi037 ceph-mon[65640]: from='client.44241 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:26:31.154 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:30 smithi037 ceph-mon[65640]: pgmap v159: 97 pgs: 97 active+clean; 341 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.9 MiB/s wr, 470 op/s 2024-09-20T22:26:31.154 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:30 smithi037 ceph-mon[65640]: from='client.? 172.21.15.37:0/4142539672' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:26:31.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:30 smithi194 ceph-mon[51725]: from='client.34302 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:26:31.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:30 smithi194 ceph-mon[51725]: from='client.44241 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:26:31.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:30 smithi194 ceph-mon[51725]: pgmap v159: 97 pgs: 97 active+clean; 341 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.9 MiB/s wr, 470 op/s 2024-09-20T22:26:31.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:30 smithi194 ceph-mon[51725]: from='client.? 172.21.15.37:0/4142539672' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:26:31.618 INFO:teuthology.orchestra.run.smithi037.stdout:HEALTH_OK 2024-09-20T22:26:31.872 INFO:teuthology.orchestra.run.smithi037.stdout: 5 216302 10.51 MB/sec execute 409 sec latency 45.848 ms 2024-09-20T22:26:32.181 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:31 smithi037 ceph-mon[65640]: from='client.34312 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:26:32.182 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:31 smithi037 ceph-mon[65640]: from='client.? 172.21.15.37:0/216854779' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T22:26:32.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:31 smithi194 ceph-mon[51725]: from='client.34312 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:26:32.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:31 smithi194 ceph-mon[51725]: from='client.? 172.21.15.37:0/216854779' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T22:26:32.872 INFO:teuthology.orchestra.run.smithi037.stdout: 5 216742 10.52 MB/sec execute 410 sec latency 49.610 ms 2024-09-20T22:26:33.222 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:32 smithi037 ceph-mon[65640]: pgmap v160: 97 pgs: 97 active+clean; 335 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 4.6 MiB/s wr, 461 op/s 2024-09-20T22:26:33.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:32 smithi194 ceph-mon[51725]: pgmap v160: 97 pgs: 97 active+clean; 335 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 4.6 MiB/s wr, 461 op/s 2024-09-20T22:26:33.872 INFO:teuthology.orchestra.run.smithi037.stdout: 5 216979 10.52 MB/sec execute 411 sec latency 63.165 ms 2024-09-20T22:26:34.022 INFO:journalctl@ceph.osd.3.smithi037.stdout:Sep 20 22:26:34 smithi037 systemd[1]: Stopping Ceph osd.3 for f9a6357a-779c-11ef-baf5-efdc52797490... 2024-09-20T22:26:34.534 INFO:journalctl@ceph.osd.3.smithi037.stdout:Sep 20 22:26:34 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-3[44726]: 2024-09-20T22:26:34.364+0000 7fb5497b4700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-20T22:26:34.534 INFO:journalctl@ceph.osd.3.smithi037.stdout:Sep 20 22:26:34 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-3[44726]: 2024-09-20T22:26:34.364+0000 7fb5497b4700 -1 osd.3 105 *** Got signal Terminated *** 2024-09-20T22:26:34.534 INFO:journalctl@ceph.osd.3.smithi037.stdout:Sep 20 22:26:34 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-3[44726]: 2024-09-20T22:26:34.364+0000 7fb5497b4700 -1 osd.3 105 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-20T22:26:34.872 INFO:teuthology.orchestra.run.smithi037.stdout: 5 217257 10.53 MB/sec execute 412 sec latency 52.899 ms 2024-09-20T22:26:35.044 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:35 smithi037 ceph-mon[65640]: pgmap v161: 97 pgs: 97 active+clean; 342 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 6.1 MiB/s wr, 601 op/s 2024-09-20T22:26:35.044 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:35 smithi037 ceph-mon[65640]: osd.3 marked itself down and dead 2024-09-20T22:26:35.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:35 smithi194 ceph-mon[51725]: pgmap v161: 97 pgs: 97 active+clean; 342 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 6.1 MiB/s wr, 601 op/s 2024-09-20T22:26:35.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:35 smithi194 ceph-mon[51725]: osd.3 marked itself down and dead 2024-09-20T22:26:35.872 INFO:teuthology.orchestra.run.smithi037.stdout: 5 217512 10.51 MB/sec execute 413 sec latency 687.791 ms 2024-09-20T22:26:35.913 INFO:journalctl@ceph.osd.3.smithi037.stdout:Sep 20 22:26:35 smithi037 podman[79136]: 2024-09-20 22:26:35.716414778 +0000 UTC m=+1.497141906 container died 3a22952c647573b7e4543b16c73686e39b42f73db05d32c8dc80aed5a385bde7 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-3, version=8, io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, com.redhat.component=centos-stream-container, distribution-scope=public, GIT_CLEAN=True, vcs-type=git, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, release=754) 2024-09-20T22:26:36.257 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:36 smithi037 ceph-mon[65640]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-20T22:26:36.258 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:36 smithi037 ceph-mon[65640]: osdmap e106: 8 total, 7 up, 8 in 2024-09-20T22:26:36.258 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:36 smithi037 ceph-mon[65640]: osdmap e107: 8 total, 7 up, 8 in 2024-09-20T22:26:36.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:36 smithi194 ceph-mon[51725]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-20T22:26:36.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:36 smithi194 ceph-mon[51725]: osdmap e106: 8 total, 7 up, 8 in 2024-09-20T22:26:36.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:36 smithi194 ceph-mon[51725]: osdmap e107: 8 total, 7 up, 8 in 2024-09-20T22:26:36.766 INFO:journalctl@ceph.osd.3.smithi037.stdout:Sep 20 22:26:36 smithi037 podman[79136]: 2024-09-20 22:26:36.699631972 +0000 UTC m=+2.480359082 container cleanup 3a22952c647573b7e4543b16c73686e39b42f73db05d32c8dc80aed5a385bde7 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-3, io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, name=centos-stream, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, build-date=2022-05-03T08:36:31.336870, release=754, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-09-20T22:26:36.766 INFO:journalctl@ceph.osd.3.smithi037.stdout:Sep 20 22:26:36 smithi037 bash[79136]: ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-3 2024-09-20T22:26:36.872 INFO:teuthology.orchestra.run.smithi037.stdout: 5 217734 10.50 MB/sec execute 414 sec latency 841.112 ms 2024-09-20T22:26:37.017 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:37 smithi037 ceph-mon[65640]: pgmap v163: 97 pgs: 16 stale+active+clean, 81 active+clean; 339 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.4 MiB/s wr, 569 op/s 2024-09-20T22:26:37.017 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:37 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:26:37.017 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:37 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:26:37.017 INFO:journalctl@ceph.osd.3.smithi037.stdout:Sep 20 22:26:36 smithi037 podman[79153]: 2024-09-20 22:26:36.994307364 +0000 UTC m=+1.274997177 container remove 3a22952c647573b7e4543b16c73686e39b42f73db05d32c8dc80aed5a385bde7 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-3, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, GIT_CLEAN=True, RELEASE=HEAD, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, version=8, name=centos-stream, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, vcs-type=git, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-09-20T22:26:37.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:37 smithi194 ceph-mon[51725]: pgmap v163: 97 pgs: 16 stale+active+clean, 81 active+clean; 339 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.4 MiB/s wr, 569 op/s 2024-09-20T22:26:37.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:37 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:26:37.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:37 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:26:37.796 INFO:journalctl@ceph.osd.3.smithi037.stdout:Sep 20 22:26:37 smithi037 podman[79240]: 2024-09-20 22:26:37.509062051 +0000 UTC m=+0.022617009 image pull cc9251b744a3faafca7c3095f79bfff81873afe823f31a03a11e015a34abcdbd quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5 2024-09-20T22:26:37.796 INFO:journalctl@ceph.osd.3.smithi037.stdout:Sep 20 22:26:37 smithi037 podman[79240]: 2024-09-20 22:26:37.663962671 +0000 UTC m=+0.177517602 container create d535d05fac869ac85d78e379413797dcf56f57020fb08f350907dd06d9201942 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-3-deactivate, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=squid-9e2ac70, org.label-schema.schema-version=1.0) 2024-09-20T22:26:37.872 INFO:teuthology.orchestra.run.smithi037.stdout: 5 217980 10.49 MB/sec execute 415 sec latency 66.026 ms 2024-09-20T22:26:38.058 INFO:journalctl@ceph.osd.3.smithi037.stdout:Sep 20 22:26:38 smithi037 podman[79240]: 2024-09-20 22:26:38.033101789 +0000 UTC m=+0.546656727 container init d535d05fac869ac85d78e379413797dcf56f57020fb08f350907dd06d9201942 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-3-deactivate, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, ceph=True, org.label-schema.build-date=20240731, RELEASE=squid-9e2ac70, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True) 2024-09-20T22:26:38.059 INFO:journalctl@ceph.osd.3.smithi037.stdout:Sep 20 22:26:38 smithi037 podman[79240]: 2024-09-20 22:26:38.038539597 +0000 UTC m=+0.552094534 container start d535d05fac869ac85d78e379413797dcf56f57020fb08f350907dd06d9201942 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-3-deactivate, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.build-date=20240731, RELEASE=squid-9e2ac70, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-09-20T22:26:38.311 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:38 smithi037 ceph-mon[65640]: Health check failed: Degraded data redundancy: 134/1620 objects degraded (8.272%), 16 pgs degraded (PG_DEGRADED) 2024-09-20T22:26:38.311 INFO:journalctl@ceph.osd.3.smithi037.stdout:Sep 20 22:26:38 smithi037 podman[79240]: 2024-09-20 22:26:38.209020038 +0000 UTC m=+0.722574973 container attach d535d05fac869ac85d78e379413797dcf56f57020fb08f350907dd06d9201942 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-3-deactivate, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-9e2ac70, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.37.2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS) 2024-09-20T22:26:38.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:38 smithi194 ceph-mon[51725]: Health check failed: Degraded data redundancy: 134/1620 objects degraded (8.272%), 16 pgs degraded (PG_DEGRADED) 2024-09-20T22:26:38.443 INFO:journalctl@ceph.osd.3.smithi037.stdout:Sep 20 22:26:38 smithi037 podman[79240]: 2024-09-20 22:26:38.352822057 +0000 UTC m=+0.866377001 container died d535d05fac869ac85d78e379413797dcf56f57020fb08f350907dd06d9201942 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-3-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-9e2ac70, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_CLEAN=True) 2024-09-20T22:26:38.872 INFO:teuthology.orchestra.run.smithi037.stdout: 5 218171 10.46 MB/sec execute 416 sec latency 64.949 ms 2024-09-20T22:26:39.079 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:39 smithi037 ceph-mon[65640]: pgmap v165: 97 pgs: 11 active+undersized, 5 stale+active+clean, 16 active+undersized+degraded, 65 active+clean; 341 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.6 KiB/s rd, 5.8 MiB/s wr, 581 op/s; 134/1620 objects degraded (8.272%) 2024-09-20T22:26:39.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:39 smithi194 ceph-mon[51725]: pgmap v165: 97 pgs: 11 active+undersized, 5 stale+active+clean, 16 active+undersized+degraded, 65 active+clean; 341 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.6 KiB/s rd, 5.8 MiB/s wr, 581 op/s; 134/1620 objects degraded (8.272%) 2024-09-20T22:26:39.622 INFO:journalctl@ceph.osd.3.smithi037.stdout:Sep 20 22:26:39 smithi037 podman[79262]: 2024-09-20 22:26:39.434569011 +0000 UTC m=+1.070860079 container remove d535d05fac869ac85d78e379413797dcf56f57020fb08f350907dd06d9201942 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-3-deactivate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , RELEASE=squid-9e2ac70, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0) 2024-09-20T22:26:39.622 INFO:journalctl@ceph.osd.3.smithi037.stdout:Sep 20 22:26:39 smithi037 systemd[1]: ceph-f9a6357a-779c-11ef-baf5-efdc52797490@osd.3.service: Deactivated successfully. 2024-09-20T22:26:39.622 INFO:journalctl@ceph.osd.3.smithi037.stdout:Sep 20 22:26:39 smithi037 systemd[1]: Stopped Ceph osd.3 for f9a6357a-779c-11ef-baf5-efdc52797490. 2024-09-20T22:26:39.622 INFO:journalctl@ceph.osd.3.smithi037.stdout:Sep 20 22:26:39 smithi037 systemd[1]: ceph-f9a6357a-779c-11ef-baf5-efdc52797490@osd.3.service: Consumed 4min 24.834s CPU time. 2024-09-20T22:26:39.874 INFO:teuthology.orchestra.run.smithi037.stdout: 5 218417 10.45 MB/sec execute 417 sec latency 87.999 ms 2024-09-20T22:26:39.874 INFO:journalctl@ceph.osd.3.smithi037.stdout:Sep 20 22:26:39 smithi037 systemd[1]: Starting Ceph osd.3 for f9a6357a-779c-11ef-baf5-efdc52797490... 2024-09-20T22:26:40.130 INFO:journalctl@ceph.osd.3.smithi037.stdout:Sep 20 22:26:40 smithi037 podman[79358]: 2024-09-20 22:26:39.960849014 +0000 UTC m=+0.022888239 image pull cc9251b744a3faafca7c3095f79bfff81873afe823f31a03a11e015a34abcdbd quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5 2024-09-20T22:26:40.130 INFO:journalctl@ceph.osd.3.smithi037.stdout:Sep 20 22:26:40 smithi037 podman[79358]: 2024-09-20 22:26:40.085314888 +0000 UTC m=+0.147354108 container create 2dd2914c85f22b4e2199cbaac2e596459868f870b2208c5b882646d121308917 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-3-activate, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-9e2ac70) 2024-09-20T22:26:40.872 INFO:teuthology.orchestra.run.smithi037.stdout: 5 218868 10.45 MB/sec execute 418 sec latency 69.855 ms 2024-09-20T22:26:40.883 INFO:journalctl@ceph.osd.3.smithi037.stdout:Sep 20 22:26:40 smithi037 podman[79358]: 2024-09-20 22:26:40.751915691 +0000 UTC m=+0.813954921 container init 2dd2914c85f22b4e2199cbaac2e596459868f870b2208c5b882646d121308917 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-3-activate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=squid-9e2ac70, org.label-schema.build-date=20240731, GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-09-20T22:26:40.884 INFO:journalctl@ceph.osd.3.smithi037.stdout:Sep 20 22:26:40 smithi037 podman[79358]: 2024-09-20 22:26:40.765579202 +0000 UTC m=+0.827618421 container start 2dd2914c85f22b4e2199cbaac2e596459868f870b2208c5b882646d121308917 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-3-activate, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, RELEASE=squid-9e2ac70, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD) 2024-09-20T22:26:41.135 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:41 smithi037 ceph-mon[65640]: pgmap v166: 97 pgs: 13 active+undersized, 2 stale+active+clean, 23 active+undersized+degraded, 59 active+clean; 340 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 3.5 KiB/s rd, 6.1 MiB/s wr, 591 op/s; 194/1581 objects degraded (12.271%) 2024-09-20T22:26:41.135 INFO:journalctl@ceph.osd.3.smithi037.stdout:Sep 20 22:26:40 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-3-activate[79372]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T22:26:41.135 INFO:journalctl@ceph.osd.3.smithi037.stdout:Sep 20 22:26:40 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-3-activate[79372]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T22:26:41.135 INFO:journalctl@ceph.osd.3.smithi037.stdout:Sep 20 22:26:40 smithi037 bash[79358]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T22:26:41.135 INFO:journalctl@ceph.osd.3.smithi037.stdout:Sep 20 22:26:40 smithi037 bash[79358]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T22:26:41.135 INFO:journalctl@ceph.osd.3.smithi037.stdout:Sep 20 22:26:40 smithi037 podman[79358]: 2024-09-20 22:26:40.990454665 +0000 UTC m=+1.052493879 container attach 2dd2914c85f22b4e2199cbaac2e596459868f870b2208c5b882646d121308917 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-3-activate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, ceph=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=squid-9e2ac70, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-09-20T22:26:41.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:41 smithi194 ceph-mon[51725]: pgmap v166: 97 pgs: 13 active+undersized, 2 stale+active+clean, 23 active+undersized+degraded, 59 active+clean; 340 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 3.5 KiB/s rd, 6.1 MiB/s wr, 591 op/s; 194/1581 objects degraded (12.271%) 2024-09-20T22:26:41.640 INFO:journalctl@ceph.osd.3.smithi037.stdout:Sep 20 22:26:41 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-3-activate[79372]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-20T22:26:41.640 INFO:journalctl@ceph.osd.3.smithi037.stdout:Sep 20 22:26:41 smithi037 bash[79358]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-20T22:26:41.640 INFO:journalctl@ceph.osd.3.smithi037.stdout:Sep 20 22:26:41 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-3-activate[79372]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-20T22:26:41.640 INFO:journalctl@ceph.osd.3.smithi037.stdout:Sep 20 22:26:41 smithi037 bash[79358]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-20T22:26:41.872 INFO:teuthology.orchestra.run.smithi037.stdout: 5 219225 10.44 MB/sec execute 419 sec latency 49.356 ms 2024-09-20T22:26:42.148 INFO:journalctl@ceph.osd.3.smithi037.stdout:Sep 20 22:26:41 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-3-activate[79372]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-20T22:26:42.148 INFO:journalctl@ceph.osd.3.smithi037.stdout:Sep 20 22:26:41 smithi037 bash[79358]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-20T22:26:42.148 INFO:journalctl@ceph.osd.3.smithi037.stdout:Sep 20 22:26:41 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-3-activate[79372]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-20T22:26:42.148 INFO:journalctl@ceph.osd.3.smithi037.stdout:Sep 20 22:26:41 smithi037 bash[79358]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-20T22:26:42.148 INFO:journalctl@ceph.osd.3.smithi037.stdout:Sep 20 22:26:41 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-3-activate[79372]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-09-20T22:26:42.149 INFO:journalctl@ceph.osd.3.smithi037.stdout:Sep 20 22:26:41 smithi037 bash[79358]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-09-20T22:26:42.149 INFO:journalctl@ceph.osd.3.smithi037.stdout:Sep 20 22:26:41 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-3-activate[79372]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-20T22:26:42.149 INFO:journalctl@ceph.osd.3.smithi037.stdout:Sep 20 22:26:41 smithi037 bash[79358]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-20T22:26:42.149 INFO:journalctl@ceph.osd.3.smithi037.stdout:Sep 20 22:26:41 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-3-activate[79372]: --> ceph-volume raw activate successful for osd ID: 3 2024-09-20T22:26:42.149 INFO:journalctl@ceph.osd.3.smithi037.stdout:Sep 20 22:26:41 smithi037 bash[79358]: --> ceph-volume raw activate successful for osd ID: 3 2024-09-20T22:26:42.149 INFO:journalctl@ceph.osd.3.smithi037.stdout:Sep 20 22:26:41 smithi037 podman[79358]: 2024-09-20 22:26:41.980650545 +0000 UTC m=+2.042689770 container died 2dd2914c85f22b4e2199cbaac2e596459868f870b2208c5b882646d121308917 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-3-activate, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True, RELEASE=squid-9e2ac70, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-20T22:26:42.872 INFO:teuthology.orchestra.run.smithi037.stdout: 5 219575 10.45 MB/sec execute 420 sec latency 42.935 ms 2024-09-20T22:26:43.166 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:43 smithi037 ceph-mon[65640]: pgmap v167: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 339 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 3.3 MiB/s wr, 287 op/s; 211/1596 objects degraded (13.221%) 2024-09-20T22:26:43.166 INFO:journalctl@ceph.osd.3.smithi037.stdout:Sep 20 22:26:43 smithi037 podman[79517]: 2024-09-20 22:26:43.081892507 +0000 UTC m=+1.092422541 container remove 2dd2914c85f22b4e2199cbaac2e596459868f870b2208c5b882646d121308917 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-3-activate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=squid-9e2ac70, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=) 2024-09-20T22:26:43.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:43 smithi194 ceph-mon[51725]: pgmap v167: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 339 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 3.3 MiB/s wr, 287 op/s; 211/1596 objects degraded (13.221%) 2024-09-20T22:26:43.416 INFO:journalctl@ceph.osd.3.smithi037.stdout:Sep 20 22:26:43 smithi037 podman[79574]: 2024-09-20 22:26:43.232007574 +0000 UTC m=+0.023535488 image pull cc9251b744a3faafca7c3095f79bfff81873afe823f31a03a11e015a34abcdbd quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5 2024-09-20T22:26:43.417 INFO:journalctl@ceph.osd.3.smithi037.stdout:Sep 20 22:26:43 smithi037 podman[79574]: 2024-09-20 22:26:43.383125576 +0000 UTC m=+0.174653481 container create 312635406c2a164621eb12033e3b5b43056fc63bb70cf3463419d85f9b735709 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-3, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , RELEASE=squid-9e2ac70, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True) 2024-09-20T22:26:43.872 INFO:teuthology.orchestra.run.smithi037.stdout: 5 219848 10.44 MB/sec execute 421 sec latency 69.369 ms 2024-09-20T22:26:43.930 INFO:journalctl@ceph.osd.3.smithi037.stdout:Sep 20 22:26:43 smithi037 podman[79574]: 2024-09-20 22:26:43.895096719 +0000 UTC m=+0.686624641 container init 312635406c2a164621eb12033e3b5b43056fc63bb70cf3463419d85f9b735709 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-3, RELEASE=squid-9e2ac70, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-20T22:26:43.930 INFO:journalctl@ceph.osd.3.smithi037.stdout:Sep 20 22:26:43 smithi037 podman[79574]: 2024-09-20 22:26:43.903013216 +0000 UTC m=+0.694541124 container start 312635406c2a164621eb12033e3b5b43056fc63bb70cf3463419d85f9b735709 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-3, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, RELEASE=squid-9e2ac70, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2) 2024-09-20T22:26:43.930 INFO:journalctl@ceph.osd.3.smithi037.stdout:Sep 20 22:26:43 smithi037 ceph-osd[79593]: -- 172.21.15.37:0/2263661722 <== mon.0 v2:172.21.15.37:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55df810181e0 con 0x55df80fee000 2024-09-20T22:26:44.189 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:44 smithi037 ceph-mon[65640]: Health check update: Degraded data redundancy: 200/1563 objects degraded (12.796%), 27 pgs degraded (PG_DEGRADED) 2024-09-20T22:26:44.189 INFO:journalctl@ceph.osd.3.smithi037.stdout:Sep 20 22:26:44 smithi037 bash[79574]: 312635406c2a164621eb12033e3b5b43056fc63bb70cf3463419d85f9b735709 2024-09-20T22:26:44.189 INFO:journalctl@ceph.osd.3.smithi037.stdout:Sep 20 22:26:44 smithi037 systemd[1]: Started Ceph osd.3 for f9a6357a-779c-11ef-baf5-efdc52797490. 2024-09-20T22:26:44.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:44 smithi194 ceph-mon[51725]: Health check update: Degraded data redundancy: 200/1563 objects degraded (12.796%), 27 pgs degraded (PG_DEGRADED) 2024-09-20T22:26:44.872 INFO:teuthology.orchestra.run.smithi037.stdout: 5 220162 10.44 MB/sec execute 422 sec latency 55.673 ms 2024-09-20T22:26:45.246 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:45 smithi037 ceph-mon[65640]: pgmap v168: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 336 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 3.9 KiB/s rd, 4.1 MiB/s wr, 460 op/s; 200/1563 objects degraded (12.796%) 2024-09-20T22:26:45.246 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:45 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:26:45.246 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:45 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:26:45.246 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:45 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:26:45.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:45 smithi194 ceph-mon[51725]: pgmap v168: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 336 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 3.9 KiB/s rd, 4.1 MiB/s wr, 460 op/s; 200/1563 objects degraded (12.796%) 2024-09-20T22:26:45.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:45 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:26:45.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:45 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:26:45.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:45 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:26:45.872 INFO:teuthology.orchestra.run.smithi037.stdout: 5 220421 10.44 MB/sec execute 423 sec latency 56.304 ms 2024-09-20T22:26:46.263 INFO:journalctl@ceph.osd.3.smithi037.stdout:Sep 20 22:26:46 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-3[79589]: 2024-09-20T22:26:46.196+0000 7f0b7d06c640 -1 Falling back to public interface 2024-09-20T22:26:46.872 INFO:teuthology.orchestra.run.smithi037.stdout: 5 220641 10.44 MB/sec execute 424 sec latency 51.253 ms 2024-09-20T22:26:47.295 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:47 smithi037 ceph-mon[65640]: pgmap v169: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 333 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.0 MiB/s wr, 430 op/s; 198/1557 objects degraded (12.717%) 2024-09-20T22:26:47.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:47 smithi194 ceph-mon[51725]: pgmap v169: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 333 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.0 MiB/s wr, 430 op/s; 198/1557 objects degraded (12.717%) 2024-09-20T22:26:47.872 INFO:teuthology.orchestra.run.smithi037.stdout: 5 220860 10.43 MB/sec execute 425 sec latency 65.161 ms 2024-09-20T22:26:48.308 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:48 smithi037 ceph-mon[65640]: pgmap v170: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 340 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.9 MiB/s wr, 501 op/s; 199/1638 objects degraded (12.149%) 2024-09-20T22:26:48.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:48 smithi194 ceph-mon[51725]: pgmap v170: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 340 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.9 MiB/s wr, 501 op/s; 199/1638 objects degraded (12.149%) 2024-09-20T22:26:48.873 INFO:teuthology.orchestra.run.smithi037.stdout: 5 221088 10.41 MB/sec execute 426 sec latency 88.294 ms 2024-09-20T22:26:49.591 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:49 smithi037 ceph-mon[65640]: Health check update: Degraded data redundancy: 199/1638 objects degraded (12.149%), 27 pgs degraded (PG_DEGRADED) 2024-09-20T22:26:49.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:49 smithi194 ceph-mon[51725]: Health check update: Degraded data redundancy: 199/1638 objects degraded (12.149%), 27 pgs degraded (PG_DEGRADED) 2024-09-20T22:26:49.872 INFO:teuthology.orchestra.run.smithi037.stdout: 5 221213 10.40 MB/sec execute 427 sec latency 93.025 ms 2024-09-20T22:26:50.615 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:50 smithi037 ceph-mon[65640]: pgmap v171: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 341 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.8 MiB/s wr, 359 op/s; 206/1656 objects degraded (12.440%) 2024-09-20T22:26:50.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:50 smithi194 ceph-mon[51725]: pgmap v171: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 341 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.8 MiB/s wr, 359 op/s; 206/1656 objects degraded (12.440%) 2024-09-20T22:26:50.873 INFO:teuthology.orchestra.run.smithi037.stdout: 5 221428 10.38 MB/sec execute 428 sec latency 130.924 ms 2024-09-20T22:26:51.873 INFO:teuthology.orchestra.run.smithi037.stdout: 5 221694 10.37 MB/sec execute 429 sec latency 56.590 ms 2024-09-20T22:26:52.873 INFO:teuthology.orchestra.run.smithi037.stdout: 5 222050 10.36 MB/sec execute 430 sec latency 70.982 ms 2024-09-20T22:26:52.910 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:52 smithi037 ceph-mon[65640]: pgmap v172: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 344 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 3.9 MiB/s wr, 415 op/s; 203/1653 objects degraded (12.281%) 2024-09-20T22:26:52.910 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:52 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:26:52.910 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:52 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:26:52.910 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:52 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:26:52.910 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:52 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:26:53.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:52 smithi194 ceph-mon[51725]: pgmap v172: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 344 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 3.9 MiB/s wr, 415 op/s; 203/1653 objects degraded (12.281%) 2024-09-20T22:26:53.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:52 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:26:53.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:52 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:26:53.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:52 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:26:53.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:52 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:26:53.873 INFO:teuthology.orchestra.run.smithi037.stdout: 5 222454 10.36 MB/sec execute 431 sec latency 80.248 ms 2024-09-20T22:26:53.918 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:53 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:26:53.918 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:53 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:26:54.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:53 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:26:54.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:53 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:26:54.672 INFO:journalctl@ceph.osd.3.smithi037.stdout:Sep 20 22:26:54 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-3[79589]: 2024-09-20T22:26:54.595+0000 7f0b7d06c640 -1 osd.3 0 read_superblock omap replica is missing. 2024-09-20T22:26:54.873 INFO:teuthology.orchestra.run.smithi037.stdout: 5 222844 10.35 MB/sec execute 432 sec latency 57.257 ms 2024-09-20T22:26:54.923 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:54 smithi037 ceph-mon[65640]: pgmap v173: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 341 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 4.8 KiB/s rd, 4.5 MiB/s wr, 465 op/s; 196/1539 objects degraded (12.736%) 2024-09-20T22:26:54.923 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:54 smithi037 ceph-mon[65640]: Health check update: Degraded data redundancy: 196/1539 objects degraded (12.736%), 27 pgs degraded (PG_DEGRADED) 2024-09-20T22:26:55.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:54 smithi194 ceph-mon[51725]: pgmap v173: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 341 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 4.8 KiB/s rd, 4.5 MiB/s wr, 465 op/s; 196/1539 objects degraded (12.736%) 2024-09-20T22:26:55.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:54 smithi194 ceph-mon[51725]: Health check update: Degraded data redundancy: 196/1539 objects degraded (12.736%), 27 pgs degraded (PG_DEGRADED) 2024-09-20T22:26:55.873 INFO:teuthology.orchestra.run.smithi037.stdout: 5 223142 10.36 MB/sec execute 433 sec latency 58.184 ms 2024-09-20T22:26:56.208 INFO:journalctl@ceph.osd.3.smithi037.stdout:Sep 20 22:26:56 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-3[79589]: 2024-09-20T22:26:56.027+0000 7f0b7d06c640 -1 osd.3 105 log_to_monitors true 2024-09-20T22:26:56.873 INFO:teuthology.orchestra.run.smithi037.stdout: 5 223548 10.36 MB/sec execute 434 sec latency 44.958 ms 2024-09-20T22:26:56.964 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:56 smithi037 ceph-mon[65640]: pgmap v174: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 335 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 3.6 MiB/s wr, 322 op/s; 191/1530 objects degraded (12.484%) 2024-09-20T22:26:56.964 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:56 smithi037 ceph-mon[65640]: from='osd.3 [v2:172.21.15.37:6826/555643789,v1:172.21.15.37:6827/555643789]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-20T22:26:56.964 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:56 smithi037 ceph-mon[65640]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-20T22:26:57.228 INFO:journalctl@ceph.osd.3.smithi037.stdout:Sep 20 22:26:56 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-3[79589]: 2024-09-20T22:26:56.993+0000 7f0b74e17640 -1 osd.3 105 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-20T22:26:57.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:56 smithi194 ceph-mon[51725]: pgmap v174: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 335 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 3.6 MiB/s wr, 322 op/s; 191/1530 objects degraded (12.484%) 2024-09-20T22:26:57.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:56 smithi194 ceph-mon[51725]: from='osd.3 [v2:172.21.15.37:6826/555643789,v1:172.21.15.37:6827/555643789]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-20T22:26:57.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:56 smithi194 ceph-mon[51725]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-20T22:26:57.873 INFO:teuthology.orchestra.run.smithi037.stdout: 5 223897 10.37 MB/sec execute 435 sec latency 47.524 ms 2024-09-20T22:26:57.986 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:57 smithi037 ceph-mon[65640]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-20T22:26:57.986 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:57 smithi037 ceph-mon[65640]: osdmap e108: 8 total, 7 up, 8 in 2024-09-20T22:26:57.986 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:57 smithi037 ceph-mon[65640]: from='osd.3 [v2:172.21.15.37:6826/555643789,v1:172.21.15.37:6827/555643789]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi037", "root=default"]}]: dispatch 2024-09-20T22:26:57.986 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:57 smithi037 ceph-mon[65640]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi037", "root=default"]}]: dispatch 2024-09-20T22:26:58.246 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:57 smithi194 ceph-mon[51725]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-20T22:26:58.246 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:57 smithi194 ceph-mon[51725]: osdmap e108: 8 total, 7 up, 8 in 2024-09-20T22:26:58.246 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:57 smithi194 ceph-mon[51725]: from='osd.3 [v2:172.21.15.37:6826/555643789,v1:172.21.15.37:6827/555643789]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi037", "root=default"]}]: dispatch 2024-09-20T22:26:58.246 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:57 smithi194 ceph-mon[51725]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi037", "root=default"]}]: dispatch 2024-09-20T22:26:58.874 INFO:teuthology.orchestra.run.smithi037.stdout: 5 224035 10.35 MB/sec execute 436 sec latency 959.027 ms 2024-09-20T22:26:59.015 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:58 smithi037 ceph-mon[65640]: pgmap v176: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 336 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 3.9 MiB/s wr, 390 op/s; 192/1578 objects degraded (12.167%) 2024-09-20T22:26:59.015 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:58 smithi037 ceph-mon[65640]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-20T22:26:59.015 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:58 smithi037 ceph-mon[65640]: osd.3 [v2:172.21.15.37:6826/555643789,v1:172.21.15.37:6827/555643789] boot 2024-09-20T22:26:59.015 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:58 smithi037 ceph-mon[65640]: osdmap e109: 8 total, 8 up, 8 in 2024-09-20T22:26:59.015 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:58 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-20T22:26:59.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:58 smithi194 ceph-mon[51725]: pgmap v176: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 336 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 3.9 MiB/s wr, 390 op/s; 192/1578 objects degraded (12.167%) 2024-09-20T22:26:59.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:58 smithi194 ceph-mon[51725]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-20T22:26:59.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:58 smithi194 ceph-mon[51725]: osd.3 [v2:172.21.15.37:6826/555643789,v1:172.21.15.37:6827/555643789] boot 2024-09-20T22:26:59.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:58 smithi194 ceph-mon[51725]: osdmap e109: 8 total, 8 up, 8 in 2024-09-20T22:26:59.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:58 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-20T22:26:59.873 INFO:teuthology.orchestra.run.smithi037.stdout: 5 224144 10.33 MB/sec execute 437 sec latency 1959.319 ms 2024-09-20T22:27:00.147 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:59 smithi037 ceph-mon[65640]: osdmap e110: 8 total, 8 up, 8 in 2024-09-20T22:27:00.147 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:59 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:27:00.147 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:59 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:27:00.147 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:59 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:27:00.147 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:59 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:27:00.147 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:59 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:27:00.147 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:59 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:27:00.147 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:59 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:27:00.148 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:59 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:27:00.148 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:59 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:27:00.148 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:59 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:27:00.148 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:59 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-20T22:27:00.148 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:59 smithi037 ceph-mon[65640]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-20T22:27:00.148 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:59 smithi037 ceph-mon[65640]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-20T22:27:00.148 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:59 smithi037 ceph-mon[65640]: Health check update: Degraded data redundancy: 192/1578 objects degraded (12.167%), 27 pgs degraded (PG_DEGRADED) 2024-09-20T22:27:00.148 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:26:59 smithi037 ceph-mon[65640]: osdmap e111: 8 total, 8 up, 8 in 2024-09-20T22:27:00.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:59 smithi194 ceph-mon[51725]: osdmap e110: 8 total, 8 up, 8 in 2024-09-20T22:27:00.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:59 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:27:00.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:59 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:27:00.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:59 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:27:00.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:59 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:27:00.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:59 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:27:00.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:59 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:27:00.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:59 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:27:00.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:59 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:27:00.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:59 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:27:00.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:59 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:27:00.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:59 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-20T22:27:00.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:59 smithi194 ceph-mon[51725]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-20T22:27:00.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:59 smithi194 ceph-mon[51725]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-20T22:27:00.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:59 smithi194 ceph-mon[51725]: Health check update: Degraded data redundancy: 192/1578 objects degraded (12.167%), 27 pgs degraded (PG_DEGRADED) 2024-09-20T22:27:00.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:26:59 smithi194 ceph-mon[51725]: osdmap e111: 8 total, 8 up, 8 in 2024-09-20T22:27:00.873 INFO:teuthology.orchestra.run.smithi037.stdout: 5 224311 10.32 MB/sec execute 438 sec latency 2056.790 ms 2024-09-20T22:27:01.267 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:01 smithi037 ceph-mon[65640]: pgmap v179: 97 pgs: 4 remapped+peering, 12 active+undersized, 21 active+undersized+degraded, 60 active+clean; 337 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.2 MiB/s wr, 383 op/s; 123/1593 objects degraded (7.721%); 4.5 KiB/s, 2 objects/s recovering 2024-09-20T22:27:01.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:01 smithi194 ceph-mon[51725]: pgmap v179: 97 pgs: 4 remapped+peering, 12 active+undersized, 21 active+undersized+degraded, 60 active+clean; 337 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.2 MiB/s wr, 383 op/s; 123/1593 objects degraded (7.721%); 4.5 KiB/s, 2 objects/s recovering 2024-09-20T22:27:01.873 INFO:teuthology.orchestra.run.smithi037.stdout: 5 224475 10.31 MB/sec execute 439 sec latency 813.833 ms 2024-09-20T22:27:02.044 INFO:teuthology.orchestra.run.smithi037.stdout:true 2024-09-20T22:27:02.275 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:02 smithi037 ceph-mon[65640]: osdmap e112: 8 total, 8 up, 8 in 2024-09-20T22:27:02.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:02 smithi194 ceph-mon[51725]: osdmap e112: 8 total, 8 up, 8 in 2024-09-20T22:27:02.872 INFO:teuthology.orchestra.run.smithi037.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T22:27:02.872 INFO:teuthology.orchestra.run.smithi037.stdout:alertmanager.smithi037 smithi037 *:9093,9094 running (7m) 10s ago 15m 25.4M - 0.25.0 c8568f914cd2 d1f99b3cd707 2024-09-20T22:27:02.872 INFO:teuthology.orchestra.run.smithi037.stdout:crash.smithi037 smithi037 running (3m) 10s ago 15m 6895k - 19.1.1-367-g9e2ac70b cc9251b744a3 77e510d8bdb1 2024-09-20T22:27:02.872 INFO:teuthology.orchestra.run.smithi037.stdout:crash.smithi194 smithi194 running (3m) 3m ago 15m 6895k - 19.1.1-367-g9e2ac70b cc9251b744a3 4d169d1309bc 2024-09-20T22:27:02.872 INFO:teuthology.orchestra.run.smithi037.stdout:grafana.smithi037 smithi037 *:3000 running (6m) 10s ago 15m 53.2M - dad864ee21e9 2306f3c2da19 2024-09-20T22:27:02.872 INFO:teuthology.orchestra.run.smithi037.stdout:mds.foofs.smithi037.sdtedi smithi037 running (9m) 10s ago 9m 37.9M - 17.2.0 e1d6a67b021e 0a7898d25b8d 2024-09-20T22:27:02.872 INFO:teuthology.orchestra.run.smithi037.stdout:mds.foofs.smithi194.lkroso smithi194 running (9m) 3m ago 9m 15.6M - 17.2.0 e1d6a67b021e 17736de30008 2024-09-20T22:27:02.872 INFO:teuthology.orchestra.run.smithi037.stdout:mgr.smithi037.lykblh smithi037 *:8443,9283,8765 running (6m) 10s ago 17m 505M - 19.1.1-367-g9e2ac70b cc9251b744a3 181e46fd9ba6 2024-09-20T22:27:02.872 INFO:teuthology.orchestra.run.smithi037.stdout:mgr.smithi194.bkcglt smithi194 *:8443,9283,8765 running (5m) 3m ago 15m 439M - 19.1.1-367-g9e2ac70b cc9251b744a3 1fa610c97517 2024-09-20T22:27:02.873 INFO:teuthology.orchestra.run.smithi037.stdout:mon.smithi037 smithi037 running (4m) 10s ago 17m 52.0M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 c0229eceb527 2024-09-20T22:27:02.873 INFO:teuthology.orchestra.run.smithi037.stdout:mon.smithi194 smithi194 running (4m) 3m ago 15m 37.2M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 97854807ba91 2024-09-20T22:27:02.873 INFO:teuthology.orchestra.run.smithi037.stdout:nfs.foo.0.0.smithi037.spabdc smithi037 *:2049 running (9m) 10s ago 9m 280M - 4.0 e1d6a67b021e 858ff9d558a2 2024-09-20T22:27:02.873 INFO:teuthology.orchestra.run.smithi037.stdout:nfs.foo.1.0.smithi194.qzjyej smithi194 *:2049 running (9m) 3m ago 9m 89.8M - 4.0 e1d6a67b021e c07fe0eb7c75 2024-09-20T22:27:02.873 INFO:teuthology.orchestra.run.smithi037.stdout:node-exporter.smithi037 smithi037 *:9100 running (6m) 10s ago 15m 15.4M - 1.5.0 0da6a335fe13 433e6aaf6137 2024-09-20T22:27:02.873 INFO:teuthology.orchestra.run.smithi037.stdout:node-exporter.smithi194 smithi194 *:9100 running (6m) 3m ago 14m 15.7M - 1.5.0 0da6a335fe13 1d30bb810ad1 2024-09-20T22:27:02.873 INFO:teuthology.orchestra.run.smithi037.stdout:osd.0 smithi037 running (2m) 10s ago 14m 367M 1937M 19.1.1-367-g9e2ac70b cc9251b744a3 c5568bd43c07 2024-09-20T22:27:02.873 INFO:teuthology.orchestra.run.smithi037.stdout:osd.1 smithi037 running (118s) 10s ago 13m 269M 1937M 19.1.1-367-g9e2ac70b cc9251b744a3 f6b715391469 2024-09-20T22:27:02.873 INFO:teuthology.orchestra.run.smithi037.stdout:osd.2 smithi037 running (68s) 10s ago 13m 211M 1937M 19.1.1-367-g9e2ac70b cc9251b744a3 0922768db2a2 2024-09-20T22:27:02.873 INFO:teuthology.orchestra.run.smithi037.stdout:osd.3 smithi037 running (19s) 10s ago 12m 12.4M 1937M 19.1.1-367-g9e2ac70b cc9251b744a3 312635406c2a 2024-09-20T22:27:02.873 INFO:teuthology.orchestra.run.smithi037.stdout:osd.4 smithi194 running (12m) 3m ago 12m 513M 2704M 17.2.0 e1d6a67b021e fa107be8915c 2024-09-20T22:27:02.873 INFO:teuthology.orchestra.run.smithi037.stdout:osd.5 smithi194 running (12m) 3m ago 12m 324M 2704M 17.2.0 e1d6a67b021e 290b21cbd131 2024-09-20T22:27:02.873 INFO:teuthology.orchestra.run.smithi037.stdout:osd.6 smithi194 running (11m) 3m ago 11m 511M 2704M 17.2.0 e1d6a67b021e 27f0720639ed 2024-09-20T22:27:02.873 INFO:teuthology.orchestra.run.smithi037.stdout:osd.7 smithi194 running (11m) 3m ago 11m 406M 2704M 17.2.0 e1d6a67b021e 62d163f4fd92 2024-09-20T22:27:02.873 INFO:teuthology.orchestra.run.smithi037.stdout:prometheus.smithi037 smithi037 *:9095 running (5m) 10s ago 14m 66.0M - 2.43.0 a07b618ecd1d 05b826dde2ff 2024-09-20T22:27:02.874 INFO:teuthology.orchestra.run.smithi037.stdout: 5 224688 10.30 MB/sec execute 440 sec latency 1031.159 ms 2024-09-20T22:27:03.292 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:03 smithi037 ceph-mon[65640]: pgmap v182: 97 pgs: 1 activating+undersized+degraded+remapped, 4 remapped+peering, 9 active+undersized, 14 active+undersized+degraded, 69 active+clean; 342 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 2.2 MiB/s wr, 271 op/s; 113/1605 objects degraded (7.040%); 62 KiB/s, 8 objects/s recovering 2024-09-20T22:27:03.292 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:03 smithi037 ceph-mon[65640]: from='client.34324 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:27:03.292 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:03 smithi037 ceph-mon[65640]: osdmap e113: 8 total, 8 up, 8 in 2024-09-20T22:27:03.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:03 smithi194 ceph-mon[51725]: pgmap v182: 97 pgs: 1 activating+undersized+degraded+remapped, 4 remapped+peering, 9 active+undersized, 14 active+undersized+degraded, 69 active+clean; 342 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 2.2 MiB/s wr, 271 op/s; 113/1605 objects degraded (7.040%); 62 KiB/s, 8 objects/s recovering 2024-09-20T22:27:03.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:03 smithi194 ceph-mon[51725]: from='client.34324 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:27:03.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:03 smithi194 ceph-mon[51725]: osdmap e113: 8 total, 8 up, 8 in 2024-09-20T22:27:03.440 INFO:teuthology.orchestra.run.smithi037.stdout:{ 2024-09-20T22:27:03.440 INFO:teuthology.orchestra.run.smithi037.stdout: "mon": { 2024-09-20T22:27:03.440 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 2 2024-09-20T22:27:03.440 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:27:03.440 INFO:teuthology.orchestra.run.smithi037.stdout: "mgr": { 2024-09-20T22:27:03.440 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 2 2024-09-20T22:27:03.440 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:27:03.440 INFO:teuthology.orchestra.run.smithi037.stdout: "osd": { 2024-09-20T22:27:03.440 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-09-20T22:27:03.440 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 4 2024-09-20T22:27:03.440 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:27:03.440 INFO:teuthology.orchestra.run.smithi037.stdout: "mds": { 2024-09-20T22:27:03.440 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T22:27:03.441 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:27:03.441 INFO:teuthology.orchestra.run.smithi037.stdout: "overall": { 2024-09-20T22:27:03.441 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-09-20T22:27:03.441 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 8 2024-09-20T22:27:03.441 INFO:teuthology.orchestra.run.smithi037.stdout: } 2024-09-20T22:27:03.441 INFO:teuthology.orchestra.run.smithi037.stdout:} 2024-09-20T22:27:03.855 INFO:teuthology.orchestra.run.smithi037.stdout:{ 2024-09-20T22:27:03.856 INFO:teuthology.orchestra.run.smithi037.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5", 2024-09-20T22:27:03.856 INFO:teuthology.orchestra.run.smithi037.stdout: "in_progress": true, 2024-09-20T22:27:03.856 INFO:teuthology.orchestra.run.smithi037.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-20T22:27:03.856 INFO:teuthology.orchestra.run.smithi037.stdout: "services_complete": [ 2024-09-20T22:27:03.856 INFO:teuthology.orchestra.run.smithi037.stdout: "mon", 2024-09-20T22:27:03.856 INFO:teuthology.orchestra.run.smithi037.stdout: "mgr", 2024-09-20T22:27:03.856 INFO:teuthology.orchestra.run.smithi037.stdout: "crash" 2024-09-20T22:27:03.856 INFO:teuthology.orchestra.run.smithi037.stdout: ], 2024-09-20T22:27:03.857 INFO:teuthology.orchestra.run.smithi037.stdout: "progress": "10/23 daemons upgraded", 2024-09-20T22:27:03.857 INFO:teuthology.orchestra.run.smithi037.stdout: "message": "Currently upgrading osd daemons", 2024-09-20T22:27:03.857 INFO:teuthology.orchestra.run.smithi037.stdout: "is_paused": false 2024-09-20T22:27:03.857 INFO:teuthology.orchestra.run.smithi037.stdout:} 2024-09-20T22:27:03.874 INFO:teuthology.orchestra.run.smithi037.stdout: 5 224875 10.29 MB/sec execute 441 sec latency 693.056 ms 2024-09-20T22:27:04.311 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:04 smithi037 ceph-mon[65640]: from='client.44261 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:27:04.311 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:04 smithi037 ceph-mon[65640]: from='client.34332 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:27:04.311 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:04 smithi037 ceph-mon[65640]: osdmap e114: 8 total, 8 up, 8 in 2024-09-20T22:27:04.311 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:04 smithi037 ceph-mon[65640]: from='client.? 172.21.15.37:0/3534215764' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:27:04.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:04 smithi194 ceph-mon[51725]: from='client.44261 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:27:04.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:04 smithi194 ceph-mon[51725]: from='client.34332 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:27:04.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:04 smithi194 ceph-mon[51725]: osdmap e114: 8 total, 8 up, 8 in 2024-09-20T22:27:04.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:04 smithi194 ceph-mon[51725]: from='client.? 172.21.15.37:0/3534215764' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:27:04.540 INFO:teuthology.orchestra.run.smithi037.stdout:HEALTH_OK 2024-09-20T22:27:04.874 INFO:teuthology.orchestra.run.smithi037.stdout: 5 225077 10.28 MB/sec execute 442 sec latency 315.965 ms 2024-09-20T22:27:05.077 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:05 smithi037 ceph-mon[65640]: from='client.44271 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:27:05.078 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:05 smithi037 ceph-mon[65640]: pgmap v185: 97 pgs: 1 active+undersized+remapped, 4 peering, 3 remapped+peering, 89 active+clean; 341 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.0 MiB/s wr, 434 op/s; 297 KiB/s, 53 objects/s recovering 2024-09-20T22:27:05.078 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:05 smithi037 ceph-mon[65640]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 113/1605 objects degraded (7.040%), 15 pgs degraded) 2024-09-20T22:27:05.078 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:05 smithi037 ceph-mon[65640]: Cluster is now healthy 2024-09-20T22:27:05.078 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:05 smithi037 ceph-mon[65640]: osdmap e115: 8 total, 8 up, 8 in 2024-09-20T22:27:05.078 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:05 smithi037 ceph-mon[65640]: from='client.? 172.21.15.37:0/1007633276' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T22:27:05.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:05 smithi194 ceph-mon[51725]: from='client.44271 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:27:05.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:05 smithi194 ceph-mon[51725]: pgmap v185: 97 pgs: 1 active+undersized+remapped, 4 peering, 3 remapped+peering, 89 active+clean; 341 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.0 MiB/s wr, 434 op/s; 297 KiB/s, 53 objects/s recovering 2024-09-20T22:27:05.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:05 smithi194 ceph-mon[51725]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 113/1605 objects degraded (7.040%), 15 pgs degraded) 2024-09-20T22:27:05.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:05 smithi194 ceph-mon[51725]: Cluster is now healthy 2024-09-20T22:27:05.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:05 smithi194 ceph-mon[51725]: osdmap e115: 8 total, 8 up, 8 in 2024-09-20T22:27:05.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:05 smithi194 ceph-mon[51725]: from='client.? 172.21.15.37:0/1007633276' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T22:27:05.874 INFO:teuthology.orchestra.run.smithi037.stdout: 5 225322 10.27 MB/sec execute 443 sec latency 49.339 ms 2024-09-20T22:27:06.874 INFO:teuthology.orchestra.run.smithi037.stdout: 5 225555 10.25 MB/sec execute 444 sec latency 86.056 ms 2024-09-20T22:27:07.109 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:07 smithi037 ceph-mon[65640]: pgmap v187: 97 pgs: 1 active+undersized+remapped, 4 peering, 1 remapped+peering, 91 active+clean; 341 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.0 MiB/s wr, 379 op/s; 685 KiB/s, 54 objects/s recovering 2024-09-20T22:27:07.109 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:07 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:27:07.109 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:07 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:27:07.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:07 smithi194 ceph-mon[51725]: pgmap v187: 97 pgs: 1 active+undersized+remapped, 4 peering, 1 remapped+peering, 91 active+clean; 341 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.0 MiB/s wr, 379 op/s; 685 KiB/s, 54 objects/s recovering 2024-09-20T22:27:07.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:07 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:27:07.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:07 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:27:07.874 INFO:teuthology.orchestra.run.smithi037.stdout: 5 225899 10.25 MB/sec execute 445 sec latency 69.635 ms 2024-09-20T22:27:08.874 INFO:teuthology.orchestra.run.smithi037.stdout: 5 226209 10.24 MB/sec execute 446 sec latency 57.921 ms 2024-09-20T22:27:09.177 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:09 smithi037 ceph-mon[65640]: pgmap v188: 97 pgs: 97 active+clean; 344 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.8 KiB/s rd, 4.2 MiB/s wr, 382 op/s; 620 KiB/s, 45 objects/s recovering 2024-09-20T22:27:09.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:09 smithi194 ceph-mon[51725]: pgmap v188: 97 pgs: 97 active+clean; 344 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.8 KiB/s rd, 4.2 MiB/s wr, 382 op/s; 620 KiB/s, 45 objects/s recovering 2024-09-20T22:27:09.874 INFO:teuthology.orchestra.run.smithi037.stdout: 5 226609 10.25 MB/sec execute 447 sec latency 49.153 ms 2024-09-20T22:27:10.874 INFO:teuthology.orchestra.run.smithi037.stdout: 5 226934 10.24 MB/sec execute 448 sec latency 53.219 ms 2024-09-20T22:27:11.205 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:11 smithi037 ceph-mon[65640]: pgmap v189: 97 pgs: 97 active+clean; 342 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 3.4 KiB/s rd, 3.5 MiB/s wr, 307 op/s; 351 KiB/s, 21 objects/s recovering 2024-09-20T22:27:11.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:11 smithi194 ceph-mon[51725]: pgmap v189: 97 pgs: 97 active+clean; 342 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 3.4 KiB/s rd, 3.5 MiB/s wr, 307 op/s; 351 KiB/s, 21 objects/s recovering 2024-09-20T22:27:11.874 INFO:teuthology.orchestra.run.smithi037.stdout: 5 227279 10.24 MB/sec execute 449 sec latency 39.059 ms 2024-09-20T22:27:12.874 INFO:teuthology.orchestra.run.smithi037.stdout: 5 227543 10.23 MB/sec execute 450 sec latency 47.358 ms 2024-09-20T22:27:13.218 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:13 smithi037 ceph-mon[65640]: pgmap v190: 97 pgs: 97 active+clean; 342 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 2.2 MiB/s wr, 230 op/s; 312 KiB/s, 8 objects/s recovering 2024-09-20T22:27:13.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:13 smithi194 ceph-mon[51725]: pgmap v190: 97 pgs: 97 active+clean; 342 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 2.2 MiB/s wr, 230 op/s; 312 KiB/s, 8 objects/s recovering 2024-09-20T22:27:13.874 INFO:teuthology.orchestra.run.smithi037.stdout: 5 227983 10.25 MB/sec execute 451 sec latency 52.230 ms 2024-09-20T22:27:14.874 INFO:teuthology.orchestra.run.smithi037.stdout: 5 228215 10.24 MB/sec execute 452 sec latency 118.757 ms 2024-09-20T22:27:15.276 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:15 smithi037 ceph-mon[65640]: pgmap v191: 97 pgs: 97 active+clean; 338 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.6 MiB/s wr, 335 op/s; 275 KiB/s, 7 objects/s recovering 2024-09-20T22:27:15.276 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:15 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-20T22:27:15.276 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:15 smithi037 ceph-mon[65640]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-20T22:27:15.276 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:15 smithi037 ceph-mon[65640]: Upgrade: osd.4 is safe to restart 2024-09-20T22:27:15.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:15 smithi194 ceph-mon[51725]: pgmap v191: 97 pgs: 97 active+clean; 338 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.6 MiB/s wr, 335 op/s; 275 KiB/s, 7 objects/s recovering 2024-09-20T22:27:15.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:15 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-20T22:27:15.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:15 smithi194 ceph-mon[51725]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-20T22:27:15.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:15 smithi194 ceph-mon[51725]: Upgrade: osd.4 is safe to restart 2024-09-20T22:27:15.877 INFO:teuthology.orchestra.run.smithi037.stdout: 5 228585 10.24 MB/sec execute 453 sec latency 47.931 ms 2024-09-20T22:27:16.877 INFO:teuthology.orchestra.run.smithi037.stdout: 5 228839 10.23 MB/sec execute 454 sec latency 93.078 ms 2024-09-20T22:27:17.058 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:16 smithi037 ceph-mon[65640]: pgmap v192: 97 pgs: 97 active+clean; 342 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 3.6 MiB/s wr, 369 op/s; 233 KiB/s, 6 objects/s recovering 2024-09-20T22:27:17.058 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:16 smithi037 ceph-mon[65640]: Upgrade: Updating osd.4 2024-09-20T22:27:17.058 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:16 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:27:17.058 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:16 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-20T22:27:17.058 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:16 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:27:17.058 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:16 smithi037 ceph-mon[65640]: Deploying daemon osd.4 on smithi194 2024-09-20T22:27:17.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:16 smithi194 ceph-mon[51725]: pgmap v192: 97 pgs: 97 active+clean; 342 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 3.6 MiB/s wr, 369 op/s; 233 KiB/s, 6 objects/s recovering 2024-09-20T22:27:17.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:16 smithi194 ceph-mon[51725]: Upgrade: Updating osd.4 2024-09-20T22:27:17.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:16 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:27:17.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:16 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-20T22:27:17.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:16 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:27:17.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:16 smithi194 ceph-mon[51725]: Deploying daemon osd.4 on smithi194 2024-09-20T22:27:17.877 INFO:teuthology.orchestra.run.smithi037.stdout: 5 229103 10.21 MB/sec execute 455 sec latency 61.875 ms 2024-09-20T22:27:18.031 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:17 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:27:18.073 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:17 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:27:18.877 INFO:teuthology.orchestra.run.smithi037.stdout: 5 229431 10.21 MB/sec execute 456 sec latency 110.246 ms 2024-09-20T22:27:19.089 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:18 smithi037 ceph-mon[65640]: pgmap v193: 97 pgs: 97 active+clean; 341 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.8 MiB/s wr, 500 op/s; 43 KiB/s, 2 objects/s recovering 2024-09-20T22:27:19.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:18 smithi194 ceph-mon[51725]: pgmap v193: 97 pgs: 97 active+clean; 341 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.8 MiB/s wr, 500 op/s; 43 KiB/s, 2 objects/s recovering 2024-09-20T22:27:19.345 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 20 22:27:19 smithi194 systemd[1]: Stopping Ceph osd.4 for f9a6357a-779c-11ef-baf5-efdc52797490... 2024-09-20T22:27:19.845 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 20 22:27:19 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-4[30172]: 2024-09-20T22:27:19.389+0000 7f0578e4f700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-20T22:27:19.845 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 20 22:27:19 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-4[30172]: 2024-09-20T22:27:19.389+0000 7f0578e4f700 -1 osd.4 115 *** Got signal Terminated *** 2024-09-20T22:27:19.845 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 20 22:27:19 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-4[30172]: 2024-09-20T22:27:19.389+0000 7f0578e4f700 -1 osd.4 115 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-20T22:27:19.877 INFO:teuthology.orchestra.run.smithi037.stdout: 5 229672 10.20 MB/sec execute 457 sec latency 71.316 ms 2024-09-20T22:27:20.127 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:19 smithi037 ceph-mon[65640]: osd.4 marked itself down and dead 2024-09-20T22:27:20.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:19 smithi194 ceph-mon[51725]: osd.4 marked itself down and dead 2024-09-20T22:27:20.772 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 20 22:27:20 smithi194 podman[56190]: 2024-09-20 22:27:20.459709841 +0000 UTC m=+1.177217221 container died fa107be8915cab35729dc7aa58bbad45fe197c32236b8d87f3b5906133bcc0dd (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-4, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, version=8, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, distribution-scope=public, com.redhat.component=centos-stream-container) 2024-09-20T22:27:20.772 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 20 22:27:20 smithi194 podman[56190]: 2024-09-20 22:27:20.771593904 +0000 UTC m=+1.489101283 container cleanup fa107be8915cab35729dc7aa58bbad45fe197c32236b8d87f3b5906133bcc0dd (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-4, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, version=8, distribution-scope=public, vendor=Red Hat, Inc., architecture=x86_64, ceph=True, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , vcs-type=git, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8) 2024-09-20T22:27:20.877 INFO:teuthology.orchestra.run.smithi037.stdout: 5 229705 10.18 MB/sec execute 458 sec latency 920.040 ms 2024-09-20T22:27:21.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:20 smithi194 ceph-mon[51725]: pgmap v194: 97 pgs: 97 active+clean; 341 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.1 MiB/s wr, 392 op/s 2024-09-20T22:27:21.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:20 smithi194 ceph-mon[51725]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-20T22:27:21.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:20 smithi194 ceph-mon[51725]: osdmap e116: 8 total, 7 up, 8 in 2024-09-20T22:27:21.095 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 20 22:27:20 smithi194 bash[56190]: ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-4 2024-09-20T22:27:21.095 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 20 22:27:20 smithi194 podman[56207]: 2024-09-20 22:27:20.981533873 +0000 UTC m=+0.523057312 container remove fa107be8915cab35729dc7aa58bbad45fe197c32236b8d87f3b5906133bcc0dd (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-4, maintainer=Guillaume Abrioux , release=754, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, ceph=True, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, version=8) 2024-09-20T22:27:21.236 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:20 smithi037 ceph-mon[65640]: pgmap v194: 97 pgs: 97 active+clean; 341 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.1 MiB/s wr, 392 op/s 2024-09-20T22:27:21.236 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:20 smithi037 ceph-mon[65640]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-20T22:27:21.236 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:20 smithi037 ceph-mon[65640]: osdmap e116: 8 total, 7 up, 8 in 2024-09-20T22:27:21.729 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 20 22:27:21 smithi194 podman[56293]: 2024-09-20 22:27:21.37681534 +0000 UTC m=+0.019677451 image pull cc9251b744a3faafca7c3095f79bfff81873afe823f31a03a11e015a34abcdbd quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5 2024-09-20T22:27:21.729 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 20 22:27:21 smithi194 podman[56293]: 2024-09-20 22:27:21.499284508 +0000 UTC m=+0.142146625 container create 2f6949f540c29109e78406d3e7ad19fb60e050f0a41e951eac513c3d72aa0a99 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-4-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-9e2ac70, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-09-20T22:27:21.877 INFO:teuthology.orchestra.run.smithi037.stdout: 5 230025 10.17 MB/sec execute 459 sec latency 1128.883 ms 2024-09-20T22:27:21.992 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 20 22:27:21 smithi194 podman[56293]: 2024-09-20 22:27:21.727832321 +0000 UTC m=+0.370694445 container init 2f6949f540c29109e78406d3e7ad19fb60e050f0a41e951eac513c3d72aa0a99 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-4-deactivate, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=squid-9e2ac70, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-20T22:27:21.992 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 20 22:27:21 smithi194 podman[56293]: 2024-09-20 22:27:21.732545552 +0000 UTC m=+0.375407668 container start 2f6949f540c29109e78406d3e7ad19fb60e050f0a41e951eac513c3d72aa0a99 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-4-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-9e2ac70, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-20T22:27:21.992 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 20 22:27:21 smithi194 podman[56293]: 2024-09-20 22:27:21.844610739 +0000 UTC m=+0.487472859 container attach 2f6949f540c29109e78406d3e7ad19fb60e050f0a41e951eac513c3d72aa0a99 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-4-deactivate, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-9e2ac70, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True) 2024-09-20T22:27:21.992 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 20 22:27:21 smithi194 podman[56293]: 2024-09-20 22:27:21.958365573 +0000 UTC m=+0.601227689 container died 2f6949f540c29109e78406d3e7ad19fb60e050f0a41e951eac513c3d72aa0a99 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-4-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=squid-9e2ac70, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True) 2024-09-20T22:27:22.243 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:21 smithi037 ceph-mon[65640]: osdmap e117: 8 total, 7 up, 8 in 2024-09-20T22:27:22.243 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:21 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.4", "id": [7, 2, 0, 4]}]: dispatch 2024-09-20T22:27:22.243 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:21 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.7", "id": [7, 4]}]: dispatch 2024-09-20T22:27:22.243 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:21 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.a", "id": [7, 4]}]: dispatch 2024-09-20T22:27:22.243 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:21 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.b", "id": [7, 1]}]: dispatch 2024-09-20T22:27:22.243 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:21 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1c", "id": [7, 4]}]: dispatch 2024-09-20T22:27:22.243 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:21 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1f", "id": [7, 2, 0, 3]}]: dispatch 2024-09-20T22:27:22.243 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:21 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.0", "id": [3, 4]}]: dispatch 2024-09-20T22:27:22.243 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:21 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [3, 2, 1, 5]}]: dispatch 2024-09-20T22:27:22.244 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:21 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:27:22.244 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:21 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:27:22.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:21 smithi194 ceph-mon[51725]: osdmap e117: 8 total, 7 up, 8 in 2024-09-20T22:27:22.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:21 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.4", "id": [7, 2, 0, 4]}]: dispatch 2024-09-20T22:27:22.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:21 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.7", "id": [7, 4]}]: dispatch 2024-09-20T22:27:22.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:21 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.a", "id": [7, 4]}]: dispatch 2024-09-20T22:27:22.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:21 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.b", "id": [7, 1]}]: dispatch 2024-09-20T22:27:22.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:21 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1c", "id": [7, 4]}]: dispatch 2024-09-20T22:27:22.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:21 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1f", "id": [7, 2, 0, 3]}]: dispatch 2024-09-20T22:27:22.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:21 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.0", "id": [3, 4]}]: dispatch 2024-09-20T22:27:22.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:21 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [3, 2, 1, 5]}]: dispatch 2024-09-20T22:27:22.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:21 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:27:22.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:21 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:27:22.877 INFO:teuthology.orchestra.run.smithi037.stdout: 5 230290 10.17 MB/sec execute 460 sec latency 56.893 ms 2024-09-20T22:27:23.115 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 20 22:27:22 smithi194 podman[56316]: 2024-09-20 22:27:22.857588935 +0000 UTC m=+0.892553249 container remove 2f6949f540c29109e78406d3e7ad19fb60e050f0a41e951eac513c3d72aa0a99 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-4-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, RELEASE=squid-9e2ac70, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-09-20T22:27:23.115 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 20 22:27:22 smithi194 systemd[1]: ceph-f9a6357a-779c-11ef-baf5-efdc52797490@osd.4.service: Deactivated successfully. 2024-09-20T22:27:23.115 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 20 22:27:22 smithi194 systemd[1]: Stopped Ceph osd.4 for f9a6357a-779c-11ef-baf5-efdc52797490. 2024-09-20T22:27:23.115 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 20 22:27:22 smithi194 systemd[1]: ceph-f9a6357a-779c-11ef-baf5-efdc52797490@osd.4.service: Consumed 4min 9.838s CPU time. 2024-09-20T22:27:23.115 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:23 smithi194 ceph-mon[51725]: pgmap v197: 97 pgs: 3 active+undersized, 12 stale+active+clean, 4 active+undersized+degraded, 78 active+clean; 340 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 4.2 MiB/s wr, 488 op/s; 27/1545 objects degraded (1.748%) 2024-09-20T22:27:23.116 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:23 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.4", "id": [7, 2, 0, 4]}]': finished 2024-09-20T22:27:23.116 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:23 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.7", "id": [7, 4]}]': finished 2024-09-20T22:27:23.116 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:23 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.a", "id": [7, 4]}]': finished 2024-09-20T22:27:23.116 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:23 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.b", "id": [7, 1]}]': finished 2024-09-20T22:27:23.116 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:23 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1c", "id": [7, 4]}]': finished 2024-09-20T22:27:23.116 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:23 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1f", "id": [7, 2, 0, 3]}]': finished 2024-09-20T22:27:23.116 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:23 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.0", "id": [3, 4]}]': finished 2024-09-20T22:27:23.116 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:23 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [3, 2, 1, 5]}]': finished 2024-09-20T22:27:23.116 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:23 smithi194 ceph-mon[51725]: osdmap e118: 8 total, 7 up, 8 in 2024-09-20T22:27:23.116 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:23 smithi194 ceph-mon[51725]: Health check failed: Degraded data redundancy: 27/1545 objects degraded (1.748%), 4 pgs degraded (PG_DEGRADED) 2024-09-20T22:27:23.268 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:23 smithi037 ceph-mon[65640]: pgmap v197: 97 pgs: 3 active+undersized, 12 stale+active+clean, 4 active+undersized+degraded, 78 active+clean; 340 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 4.2 MiB/s wr, 488 op/s; 27/1545 objects degraded (1.748%) 2024-09-20T22:27:23.268 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:23 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.4", "id": [7, 2, 0, 4]}]': finished 2024-09-20T22:27:23.268 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:23 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.7", "id": [7, 4]}]': finished 2024-09-20T22:27:23.268 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:23 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.a", "id": [7, 4]}]': finished 2024-09-20T22:27:23.268 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:23 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.b", "id": [7, 1]}]': finished 2024-09-20T22:27:23.268 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:23 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1c", "id": [7, 4]}]': finished 2024-09-20T22:27:23.268 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:23 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1f", "id": [7, 2, 0, 3]}]': finished 2024-09-20T22:27:23.268 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:23 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.0", "id": [3, 4]}]': finished 2024-09-20T22:27:23.268 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:23 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [3, 2, 1, 5]}]': finished 2024-09-20T22:27:23.268 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:23 smithi037 ceph-mon[65640]: osdmap e118: 8 total, 7 up, 8 in 2024-09-20T22:27:23.269 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:23 smithi037 ceph-mon[65640]: Health check failed: Degraded data redundancy: 27/1545 objects degraded (1.748%), 4 pgs degraded (PG_DEGRADED) 2024-09-20T22:27:23.368 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 20 22:27:23 smithi194 systemd[1]: Starting Ceph osd.4 for f9a6357a-779c-11ef-baf5-efdc52797490... 2024-09-20T22:27:23.368 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 20 22:27:23 smithi194 podman[56417]: 2024-09-20 22:27:23.239522264 +0000 UTC m=+0.020139865 image pull cc9251b744a3faafca7c3095f79bfff81873afe823f31a03a11e015a34abcdbd quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5 2024-09-20T22:27:23.368 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 20 22:27:23 smithi194 podman[56417]: 2024-09-20 22:27:23.367059099 +0000 UTC m=+0.147676684 container create 4e983c16d49a77e09874e993fe18c968822c4ade4b049d6c438c7f5bf4f912ce (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-4-activate, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-9e2ac70, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True) 2024-09-20T22:27:23.877 INFO:teuthology.orchestra.run.smithi037.stdout: 5 230564 10.16 MB/sec execute 461 sec latency 71.689 ms 2024-09-20T22:27:24.064 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 20 22:27:23 smithi194 podman[56417]: 2024-09-20 22:27:23.620083798 +0000 UTC m=+0.400701396 container init 4e983c16d49a77e09874e993fe18c968822c4ade4b049d6c438c7f5bf4f912ce (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-4-activate, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240731, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-9e2ac70, org.label-schema.schema-version=1.0) 2024-09-20T22:27:24.064 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 20 22:27:23 smithi194 podman[56417]: 2024-09-20 22:27:23.623439848 +0000 UTC m=+0.404057436 container start 4e983c16d49a77e09874e993fe18c968822c4ade4b049d6c438c7f5bf4f912ce (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-4-activate, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, ceph=True, RELEASE=squid-9e2ac70, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-09-20T22:27:24.065 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 20 22:27:23 smithi194 podman[56417]: 2024-09-20 22:27:23.72014962 +0000 UTC m=+0.500767224 container attach 4e983c16d49a77e09874e993fe18c968822c4ade4b049d6c438c7f5bf4f912ce (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-4-activate, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, RELEASE=squid-9e2ac70, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-09-20T22:27:24.065 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 20 22:27:23 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-4-activate[56431]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T22:27:24.065 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 20 22:27:23 smithi194 bash[56417]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T22:27:24.065 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 20 22:27:23 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-4-activate[56431]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T22:27:24.065 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 20 22:27:23 smithi194 bash[56417]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T22:27:24.306 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:24 smithi037 ceph-mon[65640]: osdmap e119: 8 total, 7 up, 8 in 2024-09-20T22:27:24.307 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:24 smithi037 ceph-mon[65640]: osdmap e120: 8 total, 7 up, 8 in 2024-09-20T22:27:24.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:24 smithi194 ceph-mon[51725]: osdmap e119: 8 total, 7 up, 8 in 2024-09-20T22:27:24.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:24 smithi194 ceph-mon[51725]: osdmap e120: 8 total, 7 up, 8 in 2024-09-20T22:27:24.877 INFO:teuthology.orchestra.run.smithi037.stdout: 5 230810 10.16 MB/sec execute 462 sec latency 143.685 ms 2024-09-20T22:27:25.101 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:25 smithi037 ceph-mon[65640]: pgmap v200: 97 pgs: 3 peering, 11 active+undersized, 4 stale+active+clean, 19 active+undersized+degraded, 60 active+clean; 339 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 3.5 MiB/s wr, 511 op/s; 167/1533 objects degraded (10.894%); 0 B/s, 0 objects/s recovering 2024-09-20T22:27:25.101 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:25 smithi037 ceph-mon[65640]: 2.b starting backfill to osd.1 from (0'0,0'0] MAX to 96'8816 2024-09-20T22:27:25.101 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:25 smithi037 ceph-mon[65640]: osdmap e121: 8 total, 7 up, 8 in 2024-09-20T22:27:25.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:25 smithi194 ceph-mon[51725]: pgmap v200: 97 pgs: 3 peering, 11 active+undersized, 4 stale+active+clean, 19 active+undersized+degraded, 60 active+clean; 339 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 3.5 MiB/s wr, 511 op/s; 167/1533 objects degraded (10.894%); 0 B/s, 0 objects/s recovering 2024-09-20T22:27:25.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:25 smithi194 ceph-mon[51725]: 2.b starting backfill to osd.1 from (0'0,0'0] MAX to 96'8816 2024-09-20T22:27:25.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:25 smithi194 ceph-mon[51725]: osdmap e121: 8 total, 7 up, 8 in 2024-09-20T22:27:25.345 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 20 22:27:25 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-4-activate[56431]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-20T22:27:25.345 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 20 22:27:25 smithi194 bash[56417]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-20T22:27:25.345 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 20 22:27:25 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-4-activate[56431]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-20T22:27:25.345 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 20 22:27:25 smithi194 bash[56417]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-20T22:27:25.754 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 20 22:27:25 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-4-activate[56431]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-20T22:27:25.754 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 20 22:27:25 smithi194 bash[56417]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-20T22:27:25.754 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 20 22:27:25 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-4-activate[56431]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-20T22:27:25.755 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 20 22:27:25 smithi194 bash[56417]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-20T22:27:25.755 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 20 22:27:25 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-4-activate[56431]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-09-20T22:27:25.755 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 20 22:27:25 smithi194 bash[56417]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-09-20T22:27:25.755 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 20 22:27:25 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-4-activate[56431]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-20T22:27:25.755 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 20 22:27:25 smithi194 bash[56417]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-20T22:27:25.755 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 20 22:27:25 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-4-activate[56431]: --> ceph-volume raw activate successful for osd ID: 4 2024-09-20T22:27:25.755 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 20 22:27:25 smithi194 bash[56417]: --> ceph-volume raw activate successful for osd ID: 4 2024-09-20T22:27:25.755 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 20 22:27:25 smithi194 podman[56417]: 2024-09-20 22:27:25.492130253 +0000 UTC m=+2.272747845 container died 4e983c16d49a77e09874e993fe18c968822c4ade4b049d6c438c7f5bf4f912ce (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-4-activate, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-9e2ac70, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731) 2024-09-20T22:27:25.877 INFO:teuthology.orchestra.run.smithi037.stdout: 5 231096 10.16 MB/sec execute 463 sec latency 65.004 ms 2024-09-20T22:27:26.519 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:26 smithi194 ceph-mon[51725]: osdmap e122: 8 total, 7 up, 8 in 2024-09-20T22:27:26.519 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:26 smithi194 ceph-mon[51725]: pgmap v204: 97 pgs: 1 active+clean+remapped, 3 peering, 12 active+undersized, 26 active+undersized+degraded, 55 active+clean; 341 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 2.9 MiB/s wr, 283 op/s; 237/1542 objects degraded (15.370%); 0 B/s, 0 objects/s recovering 2024-09-20T22:27:26.520 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 20 22:27:26 smithi194 podman[56570]: 2024-09-20 22:27:26.259218445 +0000 UTC m=+0.760759630 container remove 4e983c16d49a77e09874e993fe18c968822c4ade4b049d6c438c7f5bf4f912ce (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-4-activate, ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=squid-9e2ac70, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-09-20T22:27:26.520 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 20 22:27:26 smithi194 podman[56629]: 2024-09-20 22:27:26.380394785 +0000 UTC m=+0.018246334 image pull cc9251b744a3faafca7c3095f79bfff81873afe823f31a03a11e015a34abcdbd quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5 2024-09-20T22:27:26.617 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:26 smithi037 ceph-mon[65640]: osdmap e122: 8 total, 7 up, 8 in 2024-09-20T22:27:26.617 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:26 smithi037 ceph-mon[65640]: pgmap v204: 97 pgs: 1 active+clean+remapped, 3 peering, 12 active+undersized, 26 active+undersized+degraded, 55 active+clean; 341 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 2.9 MiB/s wr, 283 op/s; 237/1542 objects degraded (15.370%); 0 B/s, 0 objects/s recovering 2024-09-20T22:27:26.772 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 20 22:27:26 smithi194 podman[56629]: 2024-09-20 22:27:26.518522881 +0000 UTC m=+0.156374422 container create 641138d63e507c235a675f50a2e9c6dab458f29aa354837ee0ab3f9da58a9314 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-4, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=squid-9e2ac70, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS) 2024-09-20T22:27:26.877 INFO:teuthology.orchestra.run.smithi037.stdout: 5 231531 10.17 MB/sec execute 464 sec latency 40.025 ms 2024-09-20T22:27:27.094 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 20 22:27:26 smithi194 podman[56629]: 2024-09-20 22:27:26.771425195 +0000 UTC m=+0.409276759 container init 641138d63e507c235a675f50a2e9c6dab458f29aa354837ee0ab3f9da58a9314 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-4, RELEASE=squid-9e2ac70, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-09-20T22:27:27.095 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 20 22:27:26 smithi194 podman[56629]: 2024-09-20 22:27:26.778602867 +0000 UTC m=+0.416454407 container start 641138d63e507c235a675f50a2e9c6dab458f29aa354837ee0ab3f9da58a9314 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-4, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, RELEASE=squid-9e2ac70, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2) 2024-09-20T22:27:27.095 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 20 22:27:26 smithi194 ceph-osd[56647]: -- 172.21.15.194:0/3417131337 <== mon.1 v2:172.21.15.194:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55559cd541e0 con 0x55559bf41c00 2024-09-20T22:27:27.095 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 20 22:27:26 smithi194 bash[56629]: 641138d63e507c235a675f50a2e9c6dab458f29aa354837ee0ab3f9da58a9314 2024-09-20T22:27:27.095 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 20 22:27:26 smithi194 systemd[1]: Started Ceph osd.4 for f9a6357a-779c-11ef-baf5-efdc52797490. 2024-09-20T22:27:27.878 INFO:teuthology.orchestra.run.smithi037.stdout: 5 231914 10.17 MB/sec execute 465 sec latency 33.198 ms 2024-09-20T22:27:28.135 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:27 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:27:28.135 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:27 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:27:28.135 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:27 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:27:28.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:27 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:27:28.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:27 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:27:28.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:27 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:27:28.613 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 20 22:27:28 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-4[56643]: 2024-09-20T22:27:28.422+0000 7f8b0ff70640 -1 Falling back to public interface 2024-09-20T22:27:28.878 INFO:teuthology.orchestra.run.smithi037.stdout: 5 232352 10.17 MB/sec execute 466 sec latency 47.343 ms 2024-09-20T22:27:29.152 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:28 smithi037 ceph-mon[65640]: pgmap v205: 97 pgs: 8 active+clean+remapped, 12 active+undersized, 26 active+undersized+degraded, 51 active+clean; 350 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.1 KiB/s rd, 4.8 MiB/s wr, 450 op/s; 244/1638 objects degraded (14.896%); 26/1638 objects misplaced (1.587%); 4.9 MiB/s, 3 keys/s, 3 objects/s recovering 2024-09-20T22:27:29.153 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:28 smithi037 ceph-mon[65640]: Health check update: Degraded data redundancy: 244/1638 objects degraded (14.896%), 26 pgs degraded (PG_DEGRADED) 2024-09-20T22:27:29.304 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:28 smithi194 ceph-mon[51725]: pgmap v205: 97 pgs: 8 active+clean+remapped, 12 active+undersized, 26 active+undersized+degraded, 51 active+clean; 350 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.1 KiB/s rd, 4.8 MiB/s wr, 450 op/s; 244/1638 objects degraded (14.896%); 26/1638 objects misplaced (1.587%); 4.9 MiB/s, 3 keys/s, 3 objects/s recovering 2024-09-20T22:27:29.304 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:28 smithi194 ceph-mon[51725]: Health check update: Degraded data redundancy: 244/1638 objects degraded (14.896%), 26 pgs degraded (PG_DEGRADED) 2024-09-20T22:27:29.878 INFO:teuthology.orchestra.run.smithi037.stdout: 5 232756 10.15 MB/sec execute 467 sec latency 40.506 ms 2024-09-20T22:27:30.878 INFO:teuthology.orchestra.run.smithi037.stdout: 5 233169 10.16 MB/sec execute 468 sec latency 46.764 ms 2024-09-20T22:27:31.177 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:30 smithi037 ceph-mon[65640]: pgmap v206: 97 pgs: 8 active+clean+remapped, 12 active+undersized, 26 active+undersized+degraded, 51 active+clean; 349 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 3.3 KiB/s rd, 4.8 MiB/s wr, 399 op/s; 247/1671 objects degraded (14.782%); 26/1671 objects misplaced (1.556%); 4.0 MiB/s, 3 keys/s, 2 objects/s recovering 2024-09-20T22:27:31.177 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:30 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:27:31.177 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:30 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:27:31.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:30 smithi194 ceph-mon[51725]: pgmap v206: 97 pgs: 8 active+clean+remapped, 12 active+undersized, 26 active+undersized+degraded, 51 active+clean; 349 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 3.3 KiB/s rd, 4.8 MiB/s wr, 399 op/s; 247/1671 objects degraded (14.782%); 26/1671 objects misplaced (1.556%); 4.0 MiB/s, 3 keys/s, 2 objects/s recovering 2024-09-20T22:27:31.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:30 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:27:31.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:30 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:27:31.878 INFO:teuthology.orchestra.run.smithi037.stdout: 5 233642 10.17 MB/sec execute 469 sec latency 51.128 ms 2024-09-20T22:27:32.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:32 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:27:32.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:32 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:27:32.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:32 smithi194 ceph-mon[51725]: pgmap v207: 97 pgs: 8 active+clean+remapped, 12 active+undersized, 26 active+undersized+degraded, 51 active+clean; 348 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.2 MiB/s wr, 331 op/s; 247/1662 objects degraded (14.862%); 26/1662 objects misplaced (1.564%); 3.1 MiB/s, 2 keys/s, 2 objects/s recovering 2024-09-20T22:27:32.688 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:32 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:27:32.689 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:32 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:27:32.689 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:32 smithi037 ceph-mon[65640]: pgmap v207: 97 pgs: 8 active+clean+remapped, 12 active+undersized, 26 active+undersized+degraded, 51 active+clean; 348 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.2 MiB/s wr, 331 op/s; 247/1662 objects degraded (14.862%); 26/1662 objects misplaced (1.564%); 3.1 MiB/s, 2 keys/s, 2 objects/s recovering 2024-09-20T22:27:32.878 INFO:teuthology.orchestra.run.smithi037.stdout: 5 234026 10.17 MB/sec execute 470 sec latency 40.620 ms 2024-09-20T22:27:33.878 INFO:teuthology.orchestra.run.smithi037.stdout: 5 234466 10.17 MB/sec execute 471 sec latency 34.147 ms 2024-09-20T22:27:34.218 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:33 smithi037 ceph-mon[65640]: Health check update: Degraded data redundancy: 237/1572 objects degraded (15.076%), 26 pgs degraded (PG_DEGRADED) 2024-09-20T22:27:34.290 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:33 smithi194 ceph-mon[51725]: Health check update: Degraded data redundancy: 237/1572 objects degraded (15.076%), 26 pgs degraded (PG_DEGRADED) 2024-09-20T22:27:34.878 INFO:teuthology.orchestra.run.smithi037.stdout: 5 234849 10.18 MB/sec execute 472 sec latency 40.161 ms 2024-09-20T22:27:34.996 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:34 smithi037 ceph-mon[65640]: pgmap v208: 97 pgs: 8 active+clean+remapped, 12 active+undersized, 26 active+undersized+degraded, 51 active+clean; 344 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 5.3 MiB/s wr, 469 op/s; 237/1572 objects degraded (15.076%); 27/1572 objects misplaced (1.718%); 2.6 MiB/s, 2 keys/s, 1 objects/s recovering 2024-09-20T22:27:35.136 INFO:teuthology.orchestra.run.smithi037.stdout:true 2024-09-20T22:27:35.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:34 smithi194 ceph-mon[51725]: pgmap v208: 97 pgs: 8 active+clean+remapped, 12 active+undersized, 26 active+undersized+degraded, 51 active+clean; 344 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 5.3 MiB/s wr, 469 op/s; 237/1572 objects degraded (15.076%); 27/1572 objects misplaced (1.718%); 2.6 MiB/s, 2 keys/s, 1 objects/s recovering 2024-09-20T22:27:35.844 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 20 22:27:35 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-4[56643]: 2024-09-20T22:27:35.508+0000 7f8b0ff70640 -1 osd.4 0 read_superblock omap replica is missing. 2024-09-20T22:27:35.878 INFO:teuthology.orchestra.run.smithi037.stdout: 5 235185 10.18 MB/sec execute 473 sec latency 40.208 ms 2024-09-20T22:27:36.254 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:36 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:27:36.254 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:36 smithi037 ceph-mon[65640]: from='client.44279 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:27:36.254 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:36 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:27:36.254 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:36 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:27:36.254 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:36 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:27:36.254 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:36 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:27:36.254 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:36 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:27:36.254 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:36 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:27:36.254 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:36 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:27:36.254 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:36 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:27:36.255 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:36 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:27:36.255 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:36 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-20T22:27:36.305 INFO:teuthology.orchestra.run.smithi037.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T22:27:36.305 INFO:teuthology.orchestra.run.smithi037.stdout:alertmanager.smithi037 smithi037 *:9093,9094 running (7m) 44s ago 16m 25.4M - 0.25.0 c8568f914cd2 d1f99b3cd707 2024-09-20T22:27:36.305 INFO:teuthology.orchestra.run.smithi037.stdout:crash.smithi037 smithi037 running (4m) 44s ago 16m 6895k - 19.1.1-367-g9e2ac70b cc9251b744a3 77e510d8bdb1 2024-09-20T22:27:36.305 INFO:teuthology.orchestra.run.smithi037.stdout:crash.smithi194 smithi194 running (3m) 5s ago 15m 6895k - 19.1.1-367-g9e2ac70b cc9251b744a3 4d169d1309bc 2024-09-20T22:27:36.305 INFO:teuthology.orchestra.run.smithi037.stdout:grafana.smithi037 smithi037 *:3000 running (7m) 44s ago 16m 53.2M - dad864ee21e9 2306f3c2da19 2024-09-20T22:27:36.305 INFO:teuthology.orchestra.run.smithi037.stdout:mds.foofs.smithi037.sdtedi smithi037 running (10m) 44s ago 10m 37.9M - 17.2.0 e1d6a67b021e 0a7898d25b8d 2024-09-20T22:27:36.305 INFO:teuthology.orchestra.run.smithi037.stdout:mds.foofs.smithi194.lkroso smithi194 running (10m) 5s ago 10m 16.4M - 17.2.0 e1d6a67b021e 17736de30008 2024-09-20T22:27:36.305 INFO:teuthology.orchestra.run.smithi037.stdout:mgr.smithi037.lykblh smithi037 *:8443,9283,8765 running (6m) 44s ago 17m 505M - 19.1.1-367-g9e2ac70b cc9251b744a3 181e46fd9ba6 2024-09-20T22:27:36.305 INFO:teuthology.orchestra.run.smithi037.stdout:mgr.smithi194.bkcglt smithi194 *:8443,9283,8765 running (5m) 5s ago 15m 443M - 19.1.1-367-g9e2ac70b cc9251b744a3 1fa610c97517 2024-09-20T22:27:36.305 INFO:teuthology.orchestra.run.smithi037.stdout:mon.smithi037 smithi037 running (5m) 44s ago 17m 52.0M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 c0229eceb527 2024-09-20T22:27:36.305 INFO:teuthology.orchestra.run.smithi037.stdout:mon.smithi194 smithi194 running (4m) 5s ago 15m 45.0M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 97854807ba91 2024-09-20T22:27:36.306 INFO:teuthology.orchestra.run.smithi037.stdout:nfs.foo.0.0.smithi037.spabdc smithi037 *:2049 running (10m) 44s ago 10m 280M - 4.0 e1d6a67b021e 858ff9d558a2 2024-09-20T22:27:36.306 INFO:teuthology.orchestra.run.smithi037.stdout:nfs.foo.1.0.smithi194.qzjyej smithi194 *:2049 running (10m) 5s ago 10m 90.2M - 4.0 e1d6a67b021e c07fe0eb7c75 2024-09-20T22:27:36.306 INFO:teuthology.orchestra.run.smithi037.stdout:node-exporter.smithi037 smithi037 *:9100 running (7m) 44s ago 16m 15.4M - 1.5.0 0da6a335fe13 433e6aaf6137 2024-09-20T22:27:36.306 INFO:teuthology.orchestra.run.smithi037.stdout:node-exporter.smithi194 smithi194 *:9100 running (6m) 5s ago 15m 15.6M - 1.5.0 0da6a335fe13 1d30bb810ad1 2024-09-20T22:27:36.306 INFO:teuthology.orchestra.run.smithi037.stdout:osd.0 smithi037 running (3m) 44s ago 14m 367M 1937M 19.1.1-367-g9e2ac70b cc9251b744a3 c5568bd43c07 2024-09-20T22:27:36.306 INFO:teuthology.orchestra.run.smithi037.stdout:osd.1 smithi037 running (2m) 44s ago 14m 269M 1937M 19.1.1-367-g9e2ac70b cc9251b744a3 f6b715391469 2024-09-20T22:27:36.306 INFO:teuthology.orchestra.run.smithi037.stdout:osd.2 smithi037 running (101s) 44s ago 13m 211M 1937M 19.1.1-367-g9e2ac70b cc9251b744a3 0922768db2a2 2024-09-20T22:27:36.306 INFO:teuthology.orchestra.run.smithi037.stdout:osd.3 smithi037 running (53s) 44s ago 13m 12.4M 1937M 19.1.1-367-g9e2ac70b cc9251b744a3 312635406c2a 2024-09-20T22:27:36.306 INFO:teuthology.orchestra.run.smithi037.stdout:osd.4 smithi194 running (9s) 5s ago 13m 12.1M 2704M 19.1.1-367-g9e2ac70b cc9251b744a3 641138d63e50 2024-09-20T22:27:36.306 INFO:teuthology.orchestra.run.smithi037.stdout:osd.5 smithi194 running (12m) 5s ago 12m 547M 2704M 17.2.0 e1d6a67b021e 290b21cbd131 2024-09-20T22:27:36.306 INFO:teuthology.orchestra.run.smithi037.stdout:osd.6 smithi194 running (12m) 5s ago 12m 481M 2704M 17.2.0 e1d6a67b021e 27f0720639ed 2024-09-20T22:27:36.306 INFO:teuthology.orchestra.run.smithi037.stdout:osd.7 smithi194 running (11m) 5s ago 11m 749M 2704M 17.2.0 e1d6a67b021e 62d163f4fd92 2024-09-20T22:27:36.306 INFO:teuthology.orchestra.run.smithi037.stdout:prometheus.smithi037 smithi037 *:9095 running (5m) 44s ago 15m 66.0M - 2.43.0 a07b618ecd1d 05b826dde2ff 2024-09-20T22:27:36.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:36 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:27:36.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:36 smithi194 ceph-mon[51725]: from='client.44279 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:27:36.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:36 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:27:36.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:36 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:27:36.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:36 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:27:36.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:36 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:27:36.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:36 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:27:36.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:36 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:27:36.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:36 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:27:36.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:36 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:27:36.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:36 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:27:36.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:36 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-20T22:27:36.878 INFO:teuthology.orchestra.run.smithi037.stdout: 5 235495 10.18 MB/sec execute 474 sec latency 66.828 ms 2024-09-20T22:27:37.031 INFO:teuthology.orchestra.run.smithi037.stdout:{ 2024-09-20T22:27:37.031 INFO:teuthology.orchestra.run.smithi037.stdout: "mon": { 2024-09-20T22:27:37.031 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 2 2024-09-20T22:27:37.032 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:27:37.032 INFO:teuthology.orchestra.run.smithi037.stdout: "mgr": { 2024-09-20T22:27:37.032 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 2 2024-09-20T22:27:37.032 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:27:37.032 INFO:teuthology.orchestra.run.smithi037.stdout: "osd": { 2024-09-20T22:27:37.032 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-09-20T22:27:37.032 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 4 2024-09-20T22:27:37.032 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:27:37.032 INFO:teuthology.orchestra.run.smithi037.stdout: "mds": { 2024-09-20T22:27:37.032 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T22:27:37.032 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:27:37.032 INFO:teuthology.orchestra.run.smithi037.stdout: "overall": { 2024-09-20T22:27:37.032 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-09-20T22:27:37.032 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 8 2024-09-20T22:27:37.033 INFO:teuthology.orchestra.run.smithi037.stdout: } 2024-09-20T22:27:37.033 INFO:teuthology.orchestra.run.smithi037.stdout:} 2024-09-20T22:27:37.094 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 20 22:27:36 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-4[56643]: 2024-09-20T22:27:36.735+0000 7f8b0ff70640 -1 osd.4 115 log_to_monitors true 2024-09-20T22:27:37.247 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:37 smithi037 ceph-mon[65640]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-20T22:27:37.247 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:37 smithi037 ceph-mon[65640]: Upgrade: unsafe to stop osd(s) at this time (5 PGs are or would become offline) 2024-09-20T22:27:37.248 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:37 smithi037 ceph-mon[65640]: from='client.44283 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:27:37.248 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:37 smithi037 ceph-mon[65640]: pgmap v209: 97 pgs: 8 active+clean+remapped, 12 active+undersized, 26 active+undersized+degraded, 51 active+clean; 348 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.4 MiB/s wr, 544 op/s; 242/1596 objects degraded (15.163%); 27/1596 objects misplaced (1.692%); 2.3 MiB/s, 1 keys/s, 1 objects/s recovering 2024-09-20T22:27:37.248 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:37 smithi037 ceph-mon[65640]: from='osd.4 [v2:172.21.15.194:6800/1309564955,v1:172.21.15.194:6801/1309564955]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-20T22:27:37.248 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:37 smithi037 ceph-mon[65640]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-20T22:27:37.248 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:37 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:27:37.248 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:37 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:27:37.248 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:37 smithi037 ceph-mon[65640]: from='client.? 172.21.15.37:0/3936416729' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:27:37.443 INFO:teuthology.orchestra.run.smithi037.stdout:{ 2024-09-20T22:27:37.443 INFO:teuthology.orchestra.run.smithi037.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5", 2024-09-20T22:27:37.444 INFO:teuthology.orchestra.run.smithi037.stdout: "in_progress": true, 2024-09-20T22:27:37.444 INFO:teuthology.orchestra.run.smithi037.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-20T22:27:37.444 INFO:teuthology.orchestra.run.smithi037.stdout: "services_complete": [ 2024-09-20T22:27:37.444 INFO:teuthology.orchestra.run.smithi037.stdout: "mon", 2024-09-20T22:27:37.444 INFO:teuthology.orchestra.run.smithi037.stdout: "mgr", 2024-09-20T22:27:37.444 INFO:teuthology.orchestra.run.smithi037.stdout: "crash" 2024-09-20T22:27:37.444 INFO:teuthology.orchestra.run.smithi037.stdout: ], 2024-09-20T22:27:37.444 INFO:teuthology.orchestra.run.smithi037.stdout: "progress": "11/23 daemons upgraded", 2024-09-20T22:27:37.444 INFO:teuthology.orchestra.run.smithi037.stdout: "message": "Currently upgrading osd daemons", 2024-09-20T22:27:37.444 INFO:teuthology.orchestra.run.smithi037.stdout: "is_paused": false 2024-09-20T22:27:37.445 INFO:teuthology.orchestra.run.smithi037.stdout:} 2024-09-20T22:27:37.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:37 smithi194 ceph-mon[51725]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-20T22:27:37.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:37 smithi194 ceph-mon[51725]: Upgrade: unsafe to stop osd(s) at this time (5 PGs are or would become offline) 2024-09-20T22:27:37.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:37 smithi194 ceph-mon[51725]: from='client.44283 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:27:37.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:37 smithi194 ceph-mon[51725]: pgmap v209: 97 pgs: 8 active+clean+remapped, 12 active+undersized, 26 active+undersized+degraded, 51 active+clean; 348 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.4 MiB/s wr, 544 op/s; 242/1596 objects degraded (15.163%); 27/1596 objects misplaced (1.692%); 2.3 MiB/s, 1 keys/s, 1 objects/s recovering 2024-09-20T22:27:37.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:37 smithi194 ceph-mon[51725]: from='osd.4 [v2:172.21.15.194:6800/1309564955,v1:172.21.15.194:6801/1309564955]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-20T22:27:37.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:37 smithi194 ceph-mon[51725]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-20T22:27:37.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:37 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:27:37.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:37 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:27:37.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:37 smithi194 ceph-mon[51725]: from='client.? 172.21.15.37:0/3936416729' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:27:37.878 INFO:teuthology.orchestra.run.smithi037.stdout: 5 235697 10.17 MB/sec execute 475 sec latency 179.487 ms 2024-09-20T22:27:38.061 INFO:teuthology.orchestra.run.smithi037.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 242/1596 objects degraded (15.163%), 26 pgs degraded 2024-09-20T22:27:38.061 INFO:teuthology.orchestra.run.smithi037.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-20T22:27:38.061 INFO:teuthology.orchestra.run.smithi037.stdout: osd.4 (root=default,host=smithi194) is down 2024-09-20T22:27:38.061 INFO:teuthology.orchestra.run.smithi037.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 242/1596 objects degraded (15.163%), 26 pgs degraded 2024-09-20T22:27:38.061 INFO:teuthology.orchestra.run.smithi037.stdout: pg 2.5 is active+undersized+degraded, acting [7,0] 2024-09-20T22:27:38.061 INFO:teuthology.orchestra.run.smithi037.stdout: pg 2.6 is active+undersized+degraded, acting [1,6] 2024-09-20T22:27:38.062 INFO:teuthology.orchestra.run.smithi037.stdout: pg 2.d is active+undersized+degraded, acting [1,3] 2024-09-20T22:27:38.062 INFO:teuthology.orchestra.run.smithi037.stdout: pg 2.f is active+undersized+degraded, acting [0,7] 2024-09-20T22:27:38.062 INFO:teuthology.orchestra.run.smithi037.stdout: pg 2.11 is active+undersized+degraded, acting [6,1] 2024-09-20T22:27:38.062 INFO:teuthology.orchestra.run.smithi037.stdout: pg 2.13 is active+undersized+degraded, acting [0,3] 2024-09-20T22:27:38.062 INFO:teuthology.orchestra.run.smithi037.stdout: pg 2.18 is active+undersized+degraded, acting [5,7] 2024-09-20T22:27:38.062 INFO:teuthology.orchestra.run.smithi037.stdout: pg 2.1a is active+undersized+degraded, acting [6,7] 2024-09-20T22:27:38.062 INFO:teuthology.orchestra.run.smithi037.stdout: pg 3.1 is active+undersized+degraded, acting [0,3] 2024-09-20T22:27:38.062 INFO:teuthology.orchestra.run.smithi037.stdout: pg 3.3 is active+undersized+degraded, acting [0,6] 2024-09-20T22:27:38.062 INFO:teuthology.orchestra.run.smithi037.stdout: pg 3.6 is active+undersized+degraded, acting [0,1] 2024-09-20T22:27:38.062 INFO:teuthology.orchestra.run.smithi037.stdout: pg 3.9 is active+undersized+degraded, acting [2,7] 2024-09-20T22:27:38.062 INFO:teuthology.orchestra.run.smithi037.stdout: pg 3.a is active+undersized+degraded, acting [6,1] 2024-09-20T22:27:38.062 INFO:teuthology.orchestra.run.smithi037.stdout: pg 3.b is active+undersized+degraded, acting [3,0] 2024-09-20T22:27:38.062 INFO:teuthology.orchestra.run.smithi037.stdout: pg 3.e is active+undersized+degraded, acting [7,1] 2024-09-20T22:27:38.062 INFO:teuthology.orchestra.run.smithi037.stdout: pg 3.f is active+undersized+degraded, acting [7,0] 2024-09-20T22:27:38.063 INFO:teuthology.orchestra.run.smithi037.stdout: pg 3.11 is active+undersized+degraded, acting [7,6] 2024-09-20T22:27:38.063 INFO:teuthology.orchestra.run.smithi037.stdout: pg 3.13 is active+undersized+degraded, acting [7,2] 2024-09-20T22:27:38.063 INFO:teuthology.orchestra.run.smithi037.stdout: pg 3.14 is active+undersized+degraded, acting [7,6] 2024-09-20T22:27:38.063 INFO:teuthology.orchestra.run.smithi037.stdout: pg 3.15 is active+undersized+degraded, acting [7,3] 2024-09-20T22:27:38.063 INFO:teuthology.orchestra.run.smithi037.stdout: pg 3.19 is active+undersized+degraded, acting [1,3] 2024-09-20T22:27:38.063 INFO:teuthology.orchestra.run.smithi037.stdout: pg 3.1a is active+undersized+degraded, acting [1,2] 2024-09-20T22:27:38.063 INFO:teuthology.orchestra.run.smithi037.stdout: pg 3.1b is active+undersized+degraded, acting [0,7] 2024-09-20T22:27:38.063 INFO:teuthology.orchestra.run.smithi037.stdout: pg 3.1c is active+undersized+degraded, acting [5,1] 2024-09-20T22:27:38.063 INFO:teuthology.orchestra.run.smithi037.stdout: pg 3.1d is active+undersized+degraded, acting [5,6] 2024-09-20T22:27:38.063 INFO:teuthology.orchestra.run.smithi037.stdout: pg 4.5 is active+undersized+degraded, acting [6,7] 2024-09-20T22:27:38.094 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 20 22:27:37 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-4[56643]: 2024-09-20T22:27:37.631+0000 7f8b03d13640 -1 osd.4 115 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-20T22:27:38.315 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:38 smithi037 ceph-mon[65640]: from='client.34358 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:27:38.315 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:38 smithi037 ceph-mon[65640]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-20T22:27:38.315 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:38 smithi037 ceph-mon[65640]: osdmap e123: 8 total, 7 up, 8 in 2024-09-20T22:27:38.315 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:38 smithi037 ceph-mon[65640]: from='osd.4 [v2:172.21.15.194:6800/1309564955,v1:172.21.15.194:6801/1309564955]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2024-09-20T22:27:38.315 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:38 smithi037 ceph-mon[65640]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2024-09-20T22:27:38.315 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:38 smithi037 ceph-mon[65640]: from='osd.4 ' entity='osd.4' 2024-09-20T22:27:38.315 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:38 smithi037 ceph-mon[65640]: from='client.? 172.21.15.37:0/177028038' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T22:27:38.400 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:38 smithi194 ceph-mon[51725]: from='client.34358 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:27:38.401 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:38 smithi194 ceph-mon[51725]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-20T22:27:38.401 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:38 smithi194 ceph-mon[51725]: osdmap e123: 8 total, 7 up, 8 in 2024-09-20T22:27:38.401 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:38 smithi194 ceph-mon[51725]: from='osd.4 [v2:172.21.15.194:6800/1309564955,v1:172.21.15.194:6801/1309564955]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2024-09-20T22:27:38.401 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:38 smithi194 ceph-mon[51725]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2024-09-20T22:27:38.401 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:38 smithi194 ceph-mon[51725]: from='osd.4 ' entity='osd.4' 2024-09-20T22:27:38.401 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:38 smithi194 ceph-mon[51725]: from='client.? 172.21.15.37:0/177028038' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T22:27:38.878 INFO:teuthology.orchestra.run.smithi037.stdout: 5 236004 10.16 MB/sec execute 476 sec latency 223.306 ms 2024-09-20T22:27:39.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:39 smithi037 ceph-mon[65640]: from='client.34366 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:27:39.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:39 smithi037 ceph-mon[65640]: pgmap v211: 97 pgs: 8 active+clean+remapped, 12 active+undersized, 26 active+undersized+degraded, 51 active+clean; 351 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.9 MiB/s wr, 546 op/s; 247/1656 objects degraded (14.915%); 27/1656 objects misplaced (1.630%) 2024-09-20T22:27:39.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:39 smithi037 ceph-mon[65640]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-20T22:27:39.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:39 smithi037 ceph-mon[65640]: osd.4 [v2:172.21.15.194:6800/1309564955,v1:172.21.15.194:6801/1309564955] boot 2024-09-20T22:27:39.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:39 smithi037 ceph-mon[65640]: osdmap e124: 8 total, 8 up, 8 in 2024-09-20T22:27:39.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:39 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-20T22:27:39.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:39 smithi194 ceph-mon[51725]: from='client.34366 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:27:39.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:39 smithi194 ceph-mon[51725]: pgmap v211: 97 pgs: 8 active+clean+remapped, 12 active+undersized, 26 active+undersized+degraded, 51 active+clean; 351 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.9 MiB/s wr, 546 op/s; 247/1656 objects degraded (14.915%); 27/1656 objects misplaced (1.630%) 2024-09-20T22:27:39.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:39 smithi194 ceph-mon[51725]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-20T22:27:39.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:39 smithi194 ceph-mon[51725]: osd.4 [v2:172.21.15.194:6800/1309564955,v1:172.21.15.194:6801/1309564955] boot 2024-09-20T22:27:39.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:39 smithi194 ceph-mon[51725]: osdmap e124: 8 total, 8 up, 8 in 2024-09-20T22:27:39.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:39 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-20T22:27:39.878 INFO:teuthology.orchestra.run.smithi037.stdout: 5 236121 10.14 MB/sec execute 477 sec latency 935.085 ms 2024-09-20T22:27:40.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:40 smithi037 ceph-mon[65640]: Health check update: Degraded data redundancy: 247/1656 objects degraded (14.915%), 26 pgs degraded (PG_DEGRADED) 2024-09-20T22:27:40.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:40 smithi037 ceph-mon[65640]: osdmap e125: 8 total, 8 up, 8 in 2024-09-20T22:27:40.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:40 smithi194 ceph-mon[51725]: Health check update: Degraded data redundancy: 247/1656 objects degraded (14.915%), 26 pgs degraded (PG_DEGRADED) 2024-09-20T22:27:40.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:40 smithi194 ceph-mon[51725]: osdmap e125: 8 total, 8 up, 8 in 2024-09-20T22:27:40.878 INFO:teuthology.orchestra.run.smithi037.stdout: 5 236160 10.12 MB/sec execute 478 sec latency 1564.965 ms 2024-09-20T22:27:41.331 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:41 smithi037 ceph-mon[65640]: pgmap v214: 97 pgs: 3 remapped+peering, 8 active+clean+remapped, 1 active+recovery_wait+degraded, 11 active+undersized, 21 active+undersized+degraded, 53 active+clean; 350 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.9 MiB/s wr, 562 op/s; 192/1632 objects degraded (11.765%); 27/1632 objects misplaced (1.654%) 2024-09-20T22:27:41.331 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:41 smithi037 ceph-mon[65640]: osdmap e126: 8 total, 8 up, 8 in 2024-09-20T22:27:41.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:41 smithi194 ceph-mon[51725]: pgmap v214: 97 pgs: 3 remapped+peering, 8 active+clean+remapped, 1 active+recovery_wait+degraded, 11 active+undersized, 21 active+undersized+degraded, 53 active+clean; 350 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.9 MiB/s wr, 562 op/s; 192/1632 objects degraded (11.765%); 27/1632 objects misplaced (1.654%) 2024-09-20T22:27:41.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:41 smithi194 ceph-mon[51725]: osdmap e126: 8 total, 8 up, 8 in 2024-09-20T22:27:41.878 INFO:teuthology.orchestra.run.smithi037.stdout: 5 236547 10.12 MB/sec execute 479 sec latency 355.252 ms 2024-09-20T22:27:42.608 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:42 smithi037 ceph-mon[65640]: osdmap e127: 8 total, 8 up, 8 in 2024-09-20T22:27:42.608 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:42 smithi037 ceph-mon[65640]: pgmap v217: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped, 1 peering, 3 remapped+peering, 4 active+clean+remapped, 1 active+recovery_wait+degraded, 9 active+undersized, 17 active+undersized+degraded, 59 active+clean; 341 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 1.2 MiB/s wr, 92 op/s; 178/1584 objects degraded (11.237%); 15/1584 objects misplaced (0.947%); 5.3 MiB/s, 80 keys/s, 8 objects/s recovering 2024-09-20T22:27:42.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:42 smithi194 ceph-mon[51725]: osdmap e127: 8 total, 8 up, 8 in 2024-09-20T22:27:42.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:42 smithi194 ceph-mon[51725]: pgmap v217: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped, 1 peering, 3 remapped+peering, 4 active+clean+remapped, 1 active+recovery_wait+degraded, 9 active+undersized, 17 active+undersized+degraded, 59 active+clean; 341 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 1.2 MiB/s wr, 92 op/s; 178/1584 objects degraded (11.237%); 15/1584 objects misplaced (0.947%); 5.3 MiB/s, 80 keys/s, 8 objects/s recovering 2024-09-20T22:27:42.878 INFO:teuthology.orchestra.run.smithi037.stdout: 5 236934 10.12 MB/sec execute 480 sec latency 1016.355 ms 2024-09-20T22:27:43.765 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:43 smithi037 ceph-mon[65640]: osdmap e128: 8 total, 8 up, 8 in 2024-09-20T22:27:43.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:43 smithi194 ceph-mon[51725]: osdmap e128: 8 total, 8 up, 8 in 2024-09-20T22:27:43.878 INFO:teuthology.orchestra.run.smithi037.stdout: 5 237176 10.10 MB/sec execute 481 sec latency 854.637 ms 2024-09-20T22:27:44.554 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:44 smithi037 ceph-mon[65640]: osdmap e129: 8 total, 8 up, 8 in 2024-09-20T22:27:44.554 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:44 smithi037 ceph-mon[65640]: pgmap v220: 97 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped, 5 peering, 2 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 84 active+clean; 341 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 2.9 MiB/s wr, 362 op/s; 65/1542 objects degraded (4.215%); 8.3 MiB/s, 81 keys/s, 53 objects/s recovering 2024-09-20T22:27:44.554 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:44 smithi037 ceph-mon[65640]: Health check update: Degraded data redundancy: 178/1584 objects degraded (11.237%), 19 pgs degraded (PG_DEGRADED) 2024-09-20T22:27:44.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:44 smithi194 ceph-mon[51725]: osdmap e129: 8 total, 8 up, 8 in 2024-09-20T22:27:44.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:44 smithi194 ceph-mon[51725]: pgmap v220: 97 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped, 5 peering, 2 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 84 active+clean; 341 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 2.9 MiB/s wr, 362 op/s; 65/1542 objects degraded (4.215%); 8.3 MiB/s, 81 keys/s, 53 objects/s recovering 2024-09-20T22:27:44.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:44 smithi194 ceph-mon[51725]: Health check update: Degraded data redundancy: 178/1584 objects degraded (11.237%), 19 pgs degraded (PG_DEGRADED) 2024-09-20T22:27:44.878 INFO:teuthology.orchestra.run.smithi037.stdout: 5 237473 10.10 MB/sec execute 482 sec latency 926.968 ms 2024-09-20T22:27:45.559 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:45 smithi037 ceph-mon[65640]: osdmap e130: 8 total, 8 up, 8 in 2024-09-20T22:27:45.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:45 smithi194 ceph-mon[51725]: osdmap e130: 8 total, 8 up, 8 in 2024-09-20T22:27:45.878 INFO:teuthology.orchestra.run.smithi037.stdout: 5 237667 10.08 MB/sec execute 483 sec latency 61.076 ms 2024-09-20T22:27:46.585 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:46 smithi037 ceph-mon[65640]: pgmap v222: 97 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped, 4 peering, 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 341 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 2.9 MiB/s wr, 306 op/s; 55/1488 objects degraded (3.696%); 2.9 MiB/s, 1 keys/s, 55 objects/s recovering 2024-09-20T22:27:46.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:46 smithi194 ceph-mon[51725]: pgmap v222: 97 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped, 4 peering, 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 341 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 2.9 MiB/s wr, 306 op/s; 55/1488 objects degraded (3.696%); 2.9 MiB/s, 1 keys/s, 55 objects/s recovering 2024-09-20T22:27:46.878 INFO:teuthology.orchestra.run.smithi037.stdout: 5 237968 10.09 MB/sec execute 484 sec latency 47.641 ms 2024-09-20T22:27:47.878 INFO:teuthology.orchestra.run.smithi037.stdout: 5 238230 10.09 MB/sec execute 485 sec latency 51.326 ms 2024-09-20T22:27:48.141 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:47 smithi037 ceph-mon[65640]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 55/1488 objects degraded (3.696%), 3 pgs degraded) 2024-09-20T22:27:48.141 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:47 smithi037 ceph-mon[65640]: Cluster is now healthy 2024-09-20T22:27:48.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:47 smithi194 ceph-mon[51725]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 55/1488 objects degraded (3.696%), 3 pgs degraded) 2024-09-20T22:27:48.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:47 smithi194 ceph-mon[51725]: Cluster is now healthy 2024-09-20T22:27:48.878 INFO:teuthology.orchestra.run.smithi037.stdout: 5 238494 10.08 MB/sec execute 486 sec latency 48.359 ms 2024-09-20T22:27:49.158 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:48 smithi037 ceph-mon[65640]: pgmap v223: 97 pgs: 97 active+clean; 346 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 5.0 MiB/s wr, 609 op/s; 2.2 MiB/s, 1 keys/s, 53 objects/s recovering 2024-09-20T22:27:49.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:48 smithi194 ceph-mon[51725]: pgmap v223: 97 pgs: 97 active+clean; 346 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 5.0 MiB/s wr, 609 op/s; 2.2 MiB/s, 1 keys/s, 53 objects/s recovering 2024-09-20T22:27:49.878 INFO:teuthology.orchestra.run.smithi037.stdout: 5 238806 10.10 MB/sec execute 487 sec latency 101.675 ms 2024-09-20T22:27:50.878 INFO:teuthology.orchestra.run.smithi037.stdout: 5 239020 10.09 MB/sec execute 488 sec latency 52.911 ms 2024-09-20T22:27:51.204 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:50 smithi037 ceph-mon[65640]: pgmap v224: 97 pgs: 97 active+clean; 348 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 3.8 KiB/s rd, 3.7 MiB/s wr, 382 op/s; 1.8 MiB/s, 0 keys/s, 35 objects/s recovering 2024-09-20T22:27:51.205 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:50 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-20T22:27:51.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:50 smithi194 ceph-mon[51725]: pgmap v224: 97 pgs: 97 active+clean; 348 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 3.8 KiB/s rd, 3.7 MiB/s wr, 382 op/s; 1.8 MiB/s, 0 keys/s, 35 objects/s recovering 2024-09-20T22:27:51.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:50 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-20T22:27:51.878 INFO:teuthology.orchestra.run.smithi037.stdout: 5 239224 10.07 MB/sec execute 489 sec latency 108.088 ms 2024-09-20T22:27:52.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:52 smithi194 ceph-mon[51725]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-20T22:27:52.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:52 smithi194 ceph-mon[51725]: Upgrade: osd.5 is safe to restart 2024-09-20T22:27:52.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:52 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:27:52.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:52 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-20T22:27:52.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:52 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:27:52.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:52 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:27:52.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:52 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:27:52.223 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:51 smithi037 ceph-mon[65640]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-20T22:27:52.223 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:52 smithi037 ceph-mon[65640]: Upgrade: osd.5 is safe to restart 2024-09-20T22:27:52.223 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:52 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:27:52.223 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:52 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-20T22:27:52.223 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:52 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:27:52.223 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:52 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:27:52.223 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:52 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:27:52.878 INFO:teuthology.orchestra.run.smithi037.stdout: 5 239491 10.05 MB/sec execute 490 sec latency 69.986 ms 2024-09-20T22:27:53.233 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:53 smithi037 ceph-mon[65640]: Upgrade: Updating osd.5 2024-09-20T22:27:53.234 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:53 smithi037 ceph-mon[65640]: Deploying daemon osd.5 on smithi194 2024-09-20T22:27:53.234 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:53 smithi037 ceph-mon[65640]: pgmap v225: 97 pgs: 97 active+clean; 346 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 3.3 MiB/s wr, 358 op/s; 166 KiB/s, 16 objects/s recovering 2024-09-20T22:27:53.255 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:53 smithi194 ceph-mon[51725]: Upgrade: Updating osd.5 2024-09-20T22:27:53.255 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:53 smithi194 ceph-mon[51725]: Deploying daemon osd.5 on smithi194 2024-09-20T22:27:53.255 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:53 smithi194 ceph-mon[51725]: pgmap v225: 97 pgs: 97 active+clean; 346 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 3.3 MiB/s wr, 358 op/s; 166 KiB/s, 16 objects/s recovering 2024-09-20T22:27:53.879 INFO:teuthology.orchestra.run.smithi037.stdout: 5 239746 10.05 MB/sec execute 491 sec latency 41.631 ms 2024-09-20T22:27:54.879 INFO:teuthology.orchestra.run.smithi037.stdout: 5 239977 10.04 MB/sec execute 492 sec latency 56.850 ms 2024-09-20T22:27:55.047 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:55 smithi037 ceph-mon[65640]: pgmap v226: 97 pgs: 97 active+clean; 346 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 4.1 MiB/s wr, 388 op/s; 139 KiB/s, 13 objects/s recovering 2024-09-20T22:27:55.324 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:27:55 smithi194 systemd[1]: Stopping Ceph osd.5 for f9a6357a-779c-11ef-baf5-efdc52797490... 2024-09-20T22:27:55.324 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:55 smithi194 ceph-mon[51725]: pgmap v226: 97 pgs: 97 active+clean; 346 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 4.1 MiB/s wr, 388 op/s; 139 KiB/s, 13 objects/s recovering 2024-09-20T22:27:55.594 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:27:55 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-5[33124]: 2024-09-20T22:27:55.322+0000 7fb472470700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-20T22:27:55.595 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:27:55 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-5[33124]: 2024-09-20T22:27:55.322+0000 7fb472470700 -1 osd.5 130 *** Got signal Terminated *** 2024-09-20T22:27:55.595 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:27:55 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-5[33124]: 2024-09-20T22:27:55.322+0000 7fb472470700 -1 osd.5 130 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-20T22:27:55.879 INFO:teuthology.orchestra.run.smithi037.stdout: 5 240374 10.05 MB/sec execute 493 sec latency 61.947 ms 2024-09-20T22:27:56.083 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:56 smithi037 ceph-mon[65640]: osd.5 marked itself down and dead 2024-09-20T22:27:56.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:56 smithi194 ceph-mon[51725]: osd.5 marked itself down and dead 2024-09-20T22:27:56.879 INFO:teuthology.orchestra.run.smithi037.stdout: 5 240425 10.03 MB/sec execute 494 sec latency 836.104 ms 2024-09-20T22:27:57.030 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:27:56 smithi194 podman[58445]: 2024-09-20 22:27:56.68660679 +0000 UTC m=+1.495325080 container died 290b21cbd131cfbf29506dc39c5af7df02cc25772783cfe818b9ecf3d0e27675 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-5, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, vcs-type=git, build-date=2022-05-03T08:36:31.336870, name=centos-stream, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , version=8, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, distribution-scope=public, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream) 2024-09-20T22:27:57.287 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:57 smithi037 ceph-mon[65640]: pgmap v227: 97 pgs: 97 active+clean; 342 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 3.9 MiB/s wr, 353 op/s; 122 KiB/s, 12 objects/s recovering 2024-09-20T22:27:57.287 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:57 smithi037 ceph-mon[65640]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-20T22:27:57.287 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:57 smithi037 ceph-mon[65640]: osdmap e131: 8 total, 7 up, 8 in 2024-09-20T22:27:57.287 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:57 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:27:57.306 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:57 smithi194 ceph-mon[51725]: pgmap v227: 97 pgs: 97 active+clean; 342 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 3.9 MiB/s wr, 353 op/s; 122 KiB/s, 12 objects/s recovering 2024-09-20T22:27:57.306 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:57 smithi194 ceph-mon[51725]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-20T22:27:57.306 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:57 smithi194 ceph-mon[51725]: osdmap e131: 8 total, 7 up, 8 in 2024-09-20T22:27:57.306 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:57 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:27:57.595 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:27:57 smithi194 podman[58445]: 2024-09-20 22:27:57.304934986 +0000 UTC m=+2.113653278 container cleanup 290b21cbd131cfbf29506dc39c5af7df02cc25772783cfe818b9ecf3d0e27675 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-5, build-date=2022-05-03T08:36:31.336870, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, version=8, maintainer=Guillaume Abrioux , RELEASE=HEAD, name=centos-stream, io.buildah.version=1.19.8, GIT_BRANCH=HEAD) 2024-09-20T22:27:57.595 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:27:57 smithi194 bash[58445]: ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-5 2024-09-20T22:27:57.595 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:27:57 smithi194 podman[58463]: 2024-09-20 22:27:57.539974038 +0000 UTC m=+0.850983983 container remove 290b21cbd131cfbf29506dc39c5af7df02cc25772783cfe818b9ecf3d0e27675 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-5, GIT_CLEAN=True, RELEASE=HEAD, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, release=754) 2024-09-20T22:27:57.879 INFO:teuthology.orchestra.run.smithi037.stdout: 5 240761 10.02 MB/sec execute 495 sec latency 1028.156 ms 2024-09-20T22:27:58.048 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:58 smithi037 ceph-mon[65640]: osdmap e132: 8 total, 7 up, 8 in 2024-09-20T22:27:58.279 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:27:58 smithi194 podman[58550]: 2024-09-20 22:27:57.928277336 +0000 UTC m=+0.018897312 image pull cc9251b744a3faafca7c3095f79bfff81873afe823f31a03a11e015a34abcdbd quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5 2024-09-20T22:27:58.279 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:27:58 smithi194 podman[58550]: 2024-09-20 22:27:58.04092165 +0000 UTC m=+0.131541625 container create da7baa6ef78875b18cddba98733e496d62479ad0152252e5f428beda3e24a8e9 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-5-deactivate, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, RELEASE=squid-9e2ac70, GIT_BRANCH=HEAD) 2024-09-20T22:27:58.279 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:58 smithi194 ceph-mon[51725]: osdmap e132: 8 total, 7 up, 8 in 2024-09-20T22:27:58.594 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:27:58 smithi194 podman[58550]: 2024-09-20 22:27:58.27759965 +0000 UTC m=+0.368219649 container init da7baa6ef78875b18cddba98733e496d62479ad0152252e5f428beda3e24a8e9 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-5-deactivate, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, RELEASE=squid-9e2ac70) 2024-09-20T22:27:58.595 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:27:58 smithi194 podman[58550]: 2024-09-20 22:27:58.282345505 +0000 UTC m=+0.372965481 container start da7baa6ef78875b18cddba98733e496d62479ad0152252e5f428beda3e24a8e9 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-5-deactivate, GIT_CLEAN=True, RELEASE=squid-9e2ac70, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-09-20T22:27:58.595 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:27:58 smithi194 podman[58550]: 2024-09-20 22:27:58.377668154 +0000 UTC m=+0.468288134 container attach da7baa6ef78875b18cddba98733e496d62479ad0152252e5f428beda3e24a8e9 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-5-deactivate, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=squid-9e2ac70, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-09-20T22:27:58.595 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:27:58 smithi194 podman[58550]: 2024-09-20 22:27:58.495848134 +0000 UTC m=+0.586468110 container died da7baa6ef78875b18cddba98733e496d62479ad0152252e5f428beda3e24a8e9 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-5-deactivate, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, RELEASE=squid-9e2ac70, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-20T22:27:58.879 INFO:teuthology.orchestra.run.smithi037.stdout: 5 241087 10.01 MB/sec execute 496 sec latency 39.194 ms 2024-09-20T22:27:59.081 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:59 smithi037 ceph-mon[65640]: pgmap v230: 97 pgs: 4 active+undersized, 11 peering, 1 stale+active+clean, 10 active+undersized+degraded, 71 active+clean; 337 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.6 KiB/s rd, 4.2 MiB/s wr, 451 op/s; 73/1521 objects degraded (4.799%) 2024-09-20T22:27:59.082 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:59 smithi037 ceph-mon[65640]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-20T22:27:59.082 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:27:59 smithi037 ceph-mon[65640]: Health check failed: Degraded data redundancy: 73/1521 objects degraded (4.799%), 10 pgs degraded (PG_DEGRADED) 2024-09-20T22:27:59.341 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:59 smithi194 ceph-mon[51725]: pgmap v230: 97 pgs: 4 active+undersized, 11 peering, 1 stale+active+clean, 10 active+undersized+degraded, 71 active+clean; 337 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.6 KiB/s rd, 4.2 MiB/s wr, 451 op/s; 73/1521 objects degraded (4.799%) 2024-09-20T22:27:59.342 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:59 smithi194 ceph-mon[51725]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-20T22:27:59.342 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:27:59 smithi194 ceph-mon[51725]: Health check failed: Degraded data redundancy: 73/1521 objects degraded (4.799%), 10 pgs degraded (PG_DEGRADED) 2024-09-20T22:27:59.595 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:27:59 smithi194 podman[58573]: 2024-09-20 22:27:59.340558185 +0000 UTC m=+0.834560747 container remove da7baa6ef78875b18cddba98733e496d62479ad0152252e5f428beda3e24a8e9 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-5-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-9e2ac70, maintainer=Guillaume Abrioux ) 2024-09-20T22:27:59.595 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:27:59 smithi194 systemd[1]: ceph-f9a6357a-779c-11ef-baf5-efdc52797490@osd.5.service: Deactivated successfully. 2024-09-20T22:27:59.595 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:27:59 smithi194 systemd[1]: Stopped Ceph osd.5 for f9a6357a-779c-11ef-baf5-efdc52797490. 2024-09-20T22:27:59.595 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:27:59 smithi194 systemd[1]: ceph-f9a6357a-779c-11ef-baf5-efdc52797490@osd.5.service: Consumed 4min 48.558s CPU time. 2024-09-20T22:27:59.595 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:27:59 smithi194 systemd[1]: Starting Ceph osd.5 for f9a6357a-779c-11ef-baf5-efdc52797490... 2024-09-20T22:27:59.879 INFO:teuthology.orchestra.run.smithi037.stdout: 5 241299 10.00 MB/sec execute 497 sec latency 43.845 ms 2024-09-20T22:28:00.055 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:27:59 smithi194 podman[58676]: 2024-09-20 22:27:59.692062034 +0000 UTC m=+0.018146765 image pull cc9251b744a3faafca7c3095f79bfff81873afe823f31a03a11e015a34abcdbd quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5 2024-09-20T22:28:00.055 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:27:59 smithi194 podman[58676]: 2024-09-20 22:27:59.81737302 +0000 UTC m=+0.143457780 container create 93c969221a0a608355ad60857788a0be3d6b69be6536d5c4c32e349cd64a4fe5 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-5-activate, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, ceph=True, RELEASE=squid-9e2ac70, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2) 2024-09-20T22:28:00.055 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:28:00 smithi194 podman[58676]: 2024-09-20 22:28:00.053566491 +0000 UTC m=+0.379651223 container init 93c969221a0a608355ad60857788a0be3d6b69be6536d5c4c32e349cd64a4fe5 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-5-activate, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=squid-9e2ac70, GIT_CLEAN=True) 2024-09-20T22:28:00.344 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:28:00 smithi194 podman[58676]: 2024-09-20 22:28:00.057205215 +0000 UTC m=+0.383289941 container start 93c969221a0a608355ad60857788a0be3d6b69be6536d5c4c32e349cd64a4fe5 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-5-activate, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=squid-9e2ac70, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731) 2024-09-20T22:28:00.345 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:28:00 smithi194 podman[58676]: 2024-09-20 22:28:00.153612936 +0000 UTC m=+0.479697673 container attach 93c969221a0a608355ad60857788a0be3d6b69be6536d5c4c32e349cd64a4fe5 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-5-activate, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, RELEASE=squid-9e2ac70, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240731, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0) 2024-09-20T22:28:00.345 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:28:00 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-5-activate[58691]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T22:28:00.345 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:28:00 smithi194 bash[58676]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T22:28:00.345 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:28:00 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-5-activate[58691]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T22:28:00.345 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:28:00 smithi194 bash[58676]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T22:28:00.644 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:00 smithi037 ceph-mon[65640]: pgmap v231: 97 pgs: 5 active+undersized, 11 peering, 1 stale+active+clean, 12 active+undersized+degraded, 68 active+clean; 336 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.2 MiB/s wr, 419 op/s; 93/1527 objects degraded (6.090%) 2024-09-20T22:28:00.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:00 smithi194 ceph-mon[51725]: pgmap v231: 97 pgs: 5 active+undersized, 11 peering, 1 stale+active+clean, 12 active+undersized+degraded, 68 active+clean; 336 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.2 MiB/s wr, 419 op/s; 93/1527 objects degraded (6.090%) 2024-09-20T22:28:00.879 INFO:teuthology.orchestra.run.smithi037.stdout: 5 241731 10.02 MB/sec execute 498 sec latency 61.685 ms 2024-09-20T22:28:01.594 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:28:01 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-5-activate[58691]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-20T22:28:01.595 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:28:01 smithi194 bash[58676]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-20T22:28:01.595 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:28:01 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-5-activate[58691]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-20T22:28:01.595 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:28:01 smithi194 bash[58676]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-20T22:28:01.879 INFO:teuthology.orchestra.run.smithi037.stdout: 5 242165 10.03 MB/sec execute 499 sec latency 35.198 ms 2024-09-20T22:28:02.095 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:28:01 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-5-activate[58691]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-20T22:28:02.095 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:28:01 smithi194 bash[58676]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-20T22:28:02.095 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:28:01 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-5-activate[58691]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-20T22:28:02.095 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:28:01 smithi194 bash[58676]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-20T22:28:02.095 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:28:01 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-5-activate[58691]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-09-20T22:28:02.095 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:28:01 smithi194 bash[58676]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-09-20T22:28:02.095 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:28:01 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-5-activate[58691]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-20T22:28:02.095 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:28:01 smithi194 bash[58676]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-20T22:28:02.095 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:28:01 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-5-activate[58691]: --> ceph-volume raw activate successful for osd ID: 5 2024-09-20T22:28:02.095 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:28:01 smithi194 bash[58676]: --> ceph-volume raw activate successful for osd ID: 5 2024-09-20T22:28:02.095 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:28:01 smithi194 podman[58676]: 2024-09-20 22:28:01.649837876 +0000 UTC m=+1.975922594 container died 93c969221a0a608355ad60857788a0be3d6b69be6536d5c4c32e349cd64a4fe5 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-5-activate, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=squid-9e2ac70, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0) 2024-09-20T22:28:02.879 INFO:teuthology.orchestra.run.smithi037.stdout: 5 242439 10.03 MB/sec execute 500 sec latency 45.472 ms 2024-09-20T22:28:03.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:02 smithi194 ceph-mon[51725]: pgmap v232: 97 pgs: 9 active+undersized, 5 peering, 18 active+undersized+degraded, 65 active+clean; 339 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 3.0 MiB/s wr, 376 op/s; 124/1551 objects degraded (7.995%) 2024-09-20T22:28:03.095 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:28:02 smithi194 podman[58836]: 2024-09-20 22:28:02.816880211 +0000 UTC m=+1.155416805 container remove 93c969221a0a608355ad60857788a0be3d6b69be6536d5c4c32e349cd64a4fe5 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-5-activate, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-9e2ac70, GIT_BRANCH=HEAD) 2024-09-20T22:28:03.095 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:28:03 smithi194 podman[58892]: 2024-09-20 22:28:02.940501749 +0000 UTC m=+0.020070187 image pull cc9251b744a3faafca7c3095f79bfff81873afe823f31a03a11e015a34abcdbd quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5 2024-09-20T22:28:03.095 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:28:03 smithi194 podman[58892]: 2024-09-20 22:28:03.051265443 +0000 UTC m=+0.130833860 container create 9a2a44a8db216e2301d554f271b9c1da0661313f3af899aa5b9c3d932a1fb5b4 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-5, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_CLEAN=True, ceph=True, RELEASE=squid-9e2ac70, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-09-20T22:28:03.181 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:02 smithi037 ceph-mon[65640]: pgmap v232: 97 pgs: 9 active+undersized, 5 peering, 18 active+undersized+degraded, 65 active+clean; 339 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 3.0 MiB/s wr, 376 op/s; 124/1551 objects degraded (7.995%) 2024-09-20T22:28:03.844 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:28:03 smithi194 podman[58892]: 2024-09-20 22:28:03.463140217 +0000 UTC m=+0.542708643 container init 9a2a44a8db216e2301d554f271b9c1da0661313f3af899aa5b9c3d932a1fb5b4 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-5, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, RELEASE=squid-9e2ac70, org.label-schema.build-date=20240731) 2024-09-20T22:28:03.845 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:28:03 smithi194 podman[58892]: 2024-09-20 22:28:03.466687153 +0000 UTC m=+0.546255571 container start 9a2a44a8db216e2301d554f271b9c1da0661313f3af899aa5b9c3d932a1fb5b4 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-5, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=squid-9e2ac70, ceph=True) 2024-09-20T22:28:03.845 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:28:03 smithi194 ceph-osd[58911]: -- 172.21.15.194:0/3456095299 <== mon.1 v2:172.21.15.194:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5599db66e1e0 con 0x5599da85bc00 2024-09-20T22:28:03.845 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:28:03 smithi194 bash[58892]: 9a2a44a8db216e2301d554f271b9c1da0661313f3af899aa5b9c3d932a1fb5b4 2024-09-20T22:28:03.845 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:28:03 smithi194 systemd[1]: Started Ceph osd.5 for f9a6357a-779c-11ef-baf5-efdc52797490. 2024-09-20T22:28:03.879 INFO:teuthology.orchestra.run.smithi037.stdout: 5 242714 10.02 MB/sec execute 501 sec latency 51.717 ms 2024-09-20T22:28:04.879 INFO:teuthology.orchestra.run.smithi037.stdout: 5 242972 10.00 MB/sec execute 502 sec latency 53.949 ms 2024-09-20T22:28:04.971 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:04 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:28:04.971 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:04 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:28:04.971 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:04 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:28:04.971 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:04 smithi037 ceph-mon[65640]: pgmap v233: 97 pgs: 9 active+undersized, 23 active+undersized+degraded, 65 active+clean; 345 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.7 MiB/s wr, 454 op/s; 182/1620 objects degraded (11.235%) 2024-09-20T22:28:04.971 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:04 smithi037 ceph-mon[65640]: Health check update: Degraded data redundancy: 124/1551 objects degraded (7.995%), 18 pgs degraded (PG_DEGRADED) 2024-09-20T22:28:05.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:04 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:28:05.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:04 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:28:05.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:04 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:28:05.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:04 smithi194 ceph-mon[51725]: pgmap v233: 97 pgs: 9 active+undersized, 23 active+undersized+degraded, 65 active+clean; 345 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.7 MiB/s wr, 454 op/s; 182/1620 objects degraded (11.235%) 2024-09-20T22:28:05.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:04 smithi194 ceph-mon[51725]: Health check update: Degraded data redundancy: 124/1551 objects degraded (7.995%), 18 pgs degraded (PG_DEGRADED) 2024-09-20T22:28:05.358 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:28:05 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-5[58907]: 2024-09-20T22:28:05.099+0000 7f63ef256640 -1 Falling back to public interface 2024-09-20T22:28:05.750 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:05 smithi037 ceph-mon[65640]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-20T22:28:05.879 INFO:teuthology.orchestra.run.smithi037.stdout: 5 243215 9.99 MB/sec execute 503 sec latency 82.477 ms 2024-09-20T22:28:06.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:05 smithi194 ceph-mon[51725]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-20T22:28:06.792 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:06 smithi037 ceph-mon[65640]: pgmap v234: 97 pgs: 9 active+undersized, 23 active+undersized+degraded, 65 active+clean; 348 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.9 MiB/s wr, 537 op/s; 191/1668 objects degraded (11.451%) 2024-09-20T22:28:06.879 INFO:teuthology.orchestra.run.smithi037.stdout: 5 243376 9.98 MB/sec execute 504 sec latency 53.700 ms 2024-09-20T22:28:07.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:06 smithi194 ceph-mon[51725]: pgmap v234: 97 pgs: 9 active+undersized, 23 active+undersized+degraded, 65 active+clean; 348 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.9 MiB/s wr, 537 op/s; 191/1668 objects degraded (11.451%) 2024-09-20T22:28:07.879 INFO:teuthology.orchestra.run.smithi037.stdout: 5 243654 9.98 MB/sec execute 505 sec latency 65.854 ms 2024-09-20T22:28:08.094 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:07 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:28:08.095 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:07 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:28:08.095 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:07 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:28:08.095 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:07 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:28:08.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:07 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:28:08.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:07 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:28:08.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:07 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:28:08.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:07 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:28:08.538 INFO:teuthology.orchestra.run.smithi037.stdout:true 2024-09-20T22:28:08.879 INFO:teuthology.orchestra.run.smithi037.stdout: 5 243914 9.98 MB/sec execute 506 sec latency 92.680 ms 2024-09-20T22:28:09.134 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:08 smithi194 ceph-mon[51725]: pgmap v235: 97 pgs: 9 active+undersized, 23 active+undersized+degraded, 65 active+clean; 340 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.3 KiB/s rd, 5.1 MiB/s wr, 534 op/s; 168/1518 objects degraded (11.067%) 2024-09-20T22:28:09.134 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:08 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:28:09.134 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:08 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:28:09.136 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:08 smithi037 ceph-mon[65640]: pgmap v235: 97 pgs: 9 active+undersized, 23 active+undersized+degraded, 65 active+clean; 340 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.3 KiB/s rd, 5.1 MiB/s wr, 534 op/s; 168/1518 objects degraded (11.067%) 2024-09-20T22:28:09.136 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:08 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:28:09.137 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:08 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:28:09.358 INFO:teuthology.orchestra.run.smithi037.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T22:28:09.358 INFO:teuthology.orchestra.run.smithi037.stdout:alertmanager.smithi037 smithi037 *:9093,9094 running (8m) 77s ago 16m 25.4M - 0.25.0 c8568f914cd2 d1f99b3cd707 2024-09-20T22:28:09.358 INFO:teuthology.orchestra.run.smithi037.stdout:crash.smithi037 smithi037 running (4m) 77s ago 16m 6895k - 19.1.1-367-g9e2ac70b cc9251b744a3 77e510d8bdb1 2024-09-20T22:28:09.359 INFO:teuthology.orchestra.run.smithi037.stdout:crash.smithi194 smithi194 running (4m) 1s ago 16m 6895k - 19.1.1-367-g9e2ac70b cc9251b744a3 4d169d1309bc 2024-09-20T22:28:09.359 INFO:teuthology.orchestra.run.smithi037.stdout:grafana.smithi037 smithi037 *:3000 running (7m) 77s ago 16m 53.2M - dad864ee21e9 2306f3c2da19 2024-09-20T22:28:09.359 INFO:teuthology.orchestra.run.smithi037.stdout:mds.foofs.smithi037.sdtedi smithi037 running (11m) 77s ago 10m 37.9M - 17.2.0 e1d6a67b021e 0a7898d25b8d 2024-09-20T22:28:09.359 INFO:teuthology.orchestra.run.smithi037.stdout:mds.foofs.smithi194.lkroso smithi194 running (10m) 1s ago 10m 16.5M - 17.2.0 e1d6a67b021e 17736de30008 2024-09-20T22:28:09.359 INFO:teuthology.orchestra.run.smithi037.stdout:mgr.smithi037.lykblh smithi037 *:8443,9283,8765 running (7m) 77s ago 18m 505M - 19.1.1-367-g9e2ac70b cc9251b744a3 181e46fd9ba6 2024-09-20T22:28:09.359 INFO:teuthology.orchestra.run.smithi037.stdout:mgr.smithi194.bkcglt smithi194 *:8443,9283,8765 running (6m) 1s ago 16m 443M - 19.1.1-367-g9e2ac70b cc9251b744a3 1fa610c97517 2024-09-20T22:28:09.359 INFO:teuthology.orchestra.run.smithi037.stdout:mon.smithi037 smithi037 running (5m) 77s ago 18m 52.0M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 c0229eceb527 2024-09-20T22:28:09.359 INFO:teuthology.orchestra.run.smithi037.stdout:mon.smithi194 smithi194 running (5m) 1s ago 16m 46.1M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 97854807ba91 2024-09-20T22:28:09.359 INFO:teuthology.orchestra.run.smithi037.stdout:nfs.foo.0.0.smithi037.spabdc smithi037 *:2049 running (10m) 77s ago 10m 280M - 4.0 e1d6a67b021e 858ff9d558a2 2024-09-20T22:28:09.359 INFO:teuthology.orchestra.run.smithi037.stdout:nfs.foo.1.0.smithi194.qzjyej smithi194 *:2049 running (10m) 1s ago 10m 90.3M - 4.0 e1d6a67b021e c07fe0eb7c75 2024-09-20T22:28:09.359 INFO:teuthology.orchestra.run.smithi037.stdout:node-exporter.smithi037 smithi037 *:9100 running (7m) 77s ago 16m 15.4M - 1.5.0 0da6a335fe13 433e6aaf6137 2024-09-20T22:28:09.359 INFO:teuthology.orchestra.run.smithi037.stdout:node-exporter.smithi194 smithi194 *:9100 running (7m) 1s ago 16m 15.8M - 1.5.0 0da6a335fe13 1d30bb810ad1 2024-09-20T22:28:09.359 INFO:teuthology.orchestra.run.smithi037.stdout:osd.0 smithi037 running (3m) 77s ago 15m 367M 1937M 19.1.1-367-g9e2ac70b cc9251b744a3 c5568bd43c07 2024-09-20T22:28:09.359 INFO:teuthology.orchestra.run.smithi037.stdout:osd.1 smithi037 running (3m) 77s ago 14m 269M 1937M 19.1.1-367-g9e2ac70b cc9251b744a3 f6b715391469 2024-09-20T22:28:09.360 INFO:teuthology.orchestra.run.smithi037.stdout:osd.2 smithi037 running (2m) 77s ago 14m 211M 1937M 19.1.1-367-g9e2ac70b cc9251b744a3 0922768db2a2 2024-09-20T22:28:09.360 INFO:teuthology.orchestra.run.smithi037.stdout:osd.3 smithi037 running (86s) 77s ago 13m 12.4M 1937M 19.1.1-367-g9e2ac70b cc9251b744a3 312635406c2a 2024-09-20T22:28:09.360 INFO:teuthology.orchestra.run.smithi037.stdout:osd.4 smithi194 running (42s) 1s ago 13m 324M 2704M 19.1.1-367-g9e2ac70b cc9251b744a3 641138d63e50 2024-09-20T22:28:09.360 INFO:teuthology.orchestra.run.smithi037.stdout:osd.5 smithi194 running (6s) 1s ago 13m 13.7M 2704M 19.1.1-367-g9e2ac70b cc9251b744a3 9a2a44a8db21 2024-09-20T22:28:09.360 INFO:teuthology.orchestra.run.smithi037.stdout:osd.6 smithi194 running (12m) 1s ago 12m 501M 2704M 17.2.0 e1d6a67b021e 27f0720639ed 2024-09-20T22:28:09.360 INFO:teuthology.orchestra.run.smithi037.stdout:osd.7 smithi194 running (12m) 1s ago 12m 793M 2704M 17.2.0 e1d6a67b021e 62d163f4fd92 2024-09-20T22:28:09.360 INFO:teuthology.orchestra.run.smithi037.stdout:prometheus.smithi037 smithi037 *:9095 running (6m) 77s ago 15m 66.0M - 2.43.0 a07b618ecd1d 05b826dde2ff 2024-09-20T22:28:09.879 INFO:teuthology.orchestra.run.smithi037.stdout: 5 244238 9.97 MB/sec execute 507 sec latency 79.980 ms 2024-09-20T22:28:09.904 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:09 smithi037 ceph-mon[65640]: from='client.34376 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:28:09.904 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:09 smithi037 ceph-mon[65640]: from='client.34380 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:28:09.904 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:09 smithi037 ceph-mon[65640]: Health check update: Degraded data redundancy: 168/1518 objects degraded (11.067%), 23 pgs degraded (PG_DEGRADED) 2024-09-20T22:28:09.905 INFO:teuthology.orchestra.run.smithi037.stdout:{ 2024-09-20T22:28:09.905 INFO:teuthology.orchestra.run.smithi037.stdout: "mon": { 2024-09-20T22:28:09.905 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 2 2024-09-20T22:28:09.905 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:28:09.905 INFO:teuthology.orchestra.run.smithi037.stdout: "mgr": { 2024-09-20T22:28:09.905 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 2 2024-09-20T22:28:09.905 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:28:09.905 INFO:teuthology.orchestra.run.smithi037.stdout: "osd": { 2024-09-20T22:28:09.905 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-09-20T22:28:09.905 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 5 2024-09-20T22:28:09.905 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:28:09.905 INFO:teuthology.orchestra.run.smithi037.stdout: "mds": { 2024-09-20T22:28:09.906 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T22:28:09.906 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:28:09.906 INFO:teuthology.orchestra.run.smithi037.stdout: "overall": { 2024-09-20T22:28:09.906 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-09-20T22:28:09.906 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 9 2024-09-20T22:28:09.906 INFO:teuthology.orchestra.run.smithi037.stdout: } 2024-09-20T22:28:09.906 INFO:teuthology.orchestra.run.smithi037.stdout:} 2024-09-20T22:28:10.137 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:09 smithi194 ceph-mon[51725]: from='client.34376 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:28:10.137 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:09 smithi194 ceph-mon[51725]: from='client.34380 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:28:10.137 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:09 smithi194 ceph-mon[51725]: Health check update: Degraded data redundancy: 168/1518 objects degraded (11.067%), 23 pgs degraded (PG_DEGRADED) 2024-09-20T22:28:10.524 INFO:teuthology.orchestra.run.smithi037.stdout:{ 2024-09-20T22:28:10.524 INFO:teuthology.orchestra.run.smithi037.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5", 2024-09-20T22:28:10.524 INFO:teuthology.orchestra.run.smithi037.stdout: "in_progress": true, 2024-09-20T22:28:10.524 INFO:teuthology.orchestra.run.smithi037.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-20T22:28:10.524 INFO:teuthology.orchestra.run.smithi037.stdout: "services_complete": [ 2024-09-20T22:28:10.524 INFO:teuthology.orchestra.run.smithi037.stdout: "mon", 2024-09-20T22:28:10.524 INFO:teuthology.orchestra.run.smithi037.stdout: "mgr", 2024-09-20T22:28:10.524 INFO:teuthology.orchestra.run.smithi037.stdout: "crash" 2024-09-20T22:28:10.524 INFO:teuthology.orchestra.run.smithi037.stdout: ], 2024-09-20T22:28:10.524 INFO:teuthology.orchestra.run.smithi037.stdout: "progress": "12/23 daemons upgraded", 2024-09-20T22:28:10.524 INFO:teuthology.orchestra.run.smithi037.stdout: "message": "Currently upgrading osd daemons", 2024-09-20T22:28:10.525 INFO:teuthology.orchestra.run.smithi037.stdout: "is_paused": false 2024-09-20T22:28:10.525 INFO:teuthology.orchestra.run.smithi037.stdout:} 2024-09-20T22:28:10.879 INFO:teuthology.orchestra.run.smithi037.stdout: 5 244619 9.96 MB/sec execute 508 sec latency 60.520 ms 2024-09-20T22:28:10.909 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:10 smithi037 ceph-mon[65640]: from='client.34384 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:28:10.909 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:10 smithi037 ceph-mon[65640]: pgmap v236: 97 pgs: 9 active+undersized, 23 active+undersized+degraded, 65 active+clean; 340 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.0 MiB/s wr, 379 op/s; 168/1482 objects degraded (11.336%) 2024-09-20T22:28:10.909 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:10 smithi037 ceph-mon[65640]: from='client.? 172.21.15.37:0/259689621' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:28:11.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:10 smithi194 ceph-mon[51725]: from='client.34384 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:28:11.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:10 smithi194 ceph-mon[51725]: pgmap v236: 97 pgs: 9 active+undersized, 23 active+undersized+degraded, 65 active+clean; 340 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.0 MiB/s wr, 379 op/s; 168/1482 objects degraded (11.336%) 2024-09-20T22:28:11.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:10 smithi194 ceph-mon[51725]: from='client.? 172.21.15.37:0/259689621' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:28:11.123 INFO:teuthology.orchestra.run.smithi037.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 168/1482 objects degraded (11.336%), 23 pgs degraded 2024-09-20T22:28:11.123 INFO:teuthology.orchestra.run.smithi037.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-20T22:28:11.123 INFO:teuthology.orchestra.run.smithi037.stdout: osd.5 (root=default,host=smithi194) is down 2024-09-20T22:28:11.123 INFO:teuthology.orchestra.run.smithi037.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 168/1482 objects degraded (11.336%), 23 pgs degraded 2024-09-20T22:28:11.123 INFO:teuthology.orchestra.run.smithi037.stdout: pg 2.2 is active+undersized+degraded, acting [1,6] 2024-09-20T22:28:11.123 INFO:teuthology.orchestra.run.smithi037.stdout: pg 2.3 is active+undersized+degraded, acting [2,7] 2024-09-20T22:28:11.123 INFO:teuthology.orchestra.run.smithi037.stdout: pg 2.b is active+undersized+degraded, acting [1,4] 2024-09-20T22:28:11.123 INFO:teuthology.orchestra.run.smithi037.stdout: pg 2.c is active+undersized+degraded, acting [2,0] 2024-09-20T22:28:11.123 INFO:teuthology.orchestra.run.smithi037.stdout: pg 2.12 is active+undersized+degraded, acting [3,7] 2024-09-20T22:28:11.123 INFO:teuthology.orchestra.run.smithi037.stdout: pg 2.14 is active+undersized+degraded, acting [6,3] 2024-09-20T22:28:11.123 INFO:teuthology.orchestra.run.smithi037.stdout: pg 2.15 is active+undersized+degraded, acting [1,0] 2024-09-20T22:28:11.124 INFO:teuthology.orchestra.run.smithi037.stdout: pg 2.16 is active+undersized+degraded, acting [6,2] 2024-09-20T22:28:11.124 INFO:teuthology.orchestra.run.smithi037.stdout: pg 2.17 is active+undersized+degraded, acting [6,2] 2024-09-20T22:28:11.124 INFO:teuthology.orchestra.run.smithi037.stdout: pg 2.18 is active+undersized+degraded, acting [4,7] 2024-09-20T22:28:11.124 INFO:teuthology.orchestra.run.smithi037.stdout: pg 2.1c is active+undersized+degraded, acting [4,2] 2024-09-20T22:28:11.124 INFO:teuthology.orchestra.run.smithi037.stdout: pg 2.1e is active+undersized+degraded, acting [3,0] 2024-09-20T22:28:11.124 INFO:teuthology.orchestra.run.smithi037.stdout: pg 3.2 is active+undersized+degraded, acting [3,6] 2024-09-20T22:28:11.124 INFO:teuthology.orchestra.run.smithi037.stdout: pg 3.4 is active+undersized+degraded, acting [1,2] 2024-09-20T22:28:11.124 INFO:teuthology.orchestra.run.smithi037.stdout: pg 3.5 is active+undersized+degraded, acting [3,2] 2024-09-20T22:28:11.124 INFO:teuthology.orchestra.run.smithi037.stdout: pg 3.c is active+undersized+degraded, acting [3,6] 2024-09-20T22:28:11.124 INFO:teuthology.orchestra.run.smithi037.stdout: pg 3.d is active+undersized+degraded, acting [7,6] 2024-09-20T22:28:11.124 INFO:teuthology.orchestra.run.smithi037.stdout: pg 3.10 is active+undersized+degraded, acting [6,0] 2024-09-20T22:28:11.124 INFO:teuthology.orchestra.run.smithi037.stdout: pg 3.16 is active+undersized+degraded, acting [7,1] 2024-09-20T22:28:11.124 INFO:teuthology.orchestra.run.smithi037.stdout: pg 3.17 is active+undersized+degraded, acting [0,3] 2024-09-20T22:28:11.124 INFO:teuthology.orchestra.run.smithi037.stdout: pg 3.1c is active+undersized+degraded, acting [4,1] 2024-09-20T22:28:11.125 INFO:teuthology.orchestra.run.smithi037.stdout: pg 3.1d is active+undersized+degraded, acting [4,6] 2024-09-20T22:28:11.125 INFO:teuthology.orchestra.run.smithi037.stdout: pg 3.1f is active+undersized+degraded, acting [0,2] 2024-09-20T22:28:11.879 INFO:teuthology.orchestra.run.smithi037.stdout: 5 244887 9.96 MB/sec execute 509 sec latency 54.739 ms 2024-09-20T22:28:11.945 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:11 smithi037 ceph-mon[65640]: from='client.34392 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:28:11.945 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:11 smithi037 ceph-mon[65640]: from='client.? 172.21.15.37:0/3286408878' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T22:28:12.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:11 smithi194 ceph-mon[51725]: from='client.34392 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:28:12.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:11 smithi194 ceph-mon[51725]: from='client.? 172.21.15.37:0/3286408878' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T22:28:12.879 INFO:teuthology.orchestra.run.smithi037.stdout: 5 245358 9.97 MB/sec execute 510 sec latency 40.101 ms 2024-09-20T22:28:12.958 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:12 smithi037 ceph-mon[65640]: pgmap v237: 97 pgs: 9 active+undersized, 23 active+undersized+degraded, 65 active+clean; 333 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.0 MiB/s wr, 414 op/s; 168/1470 objects degraded (11.429%) 2024-09-20T22:28:12.958 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:12 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:28:12.958 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:12 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:28:12.958 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:12 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:28:12.958 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:12 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:28:12.958 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:12 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:28:12.958 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:12 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:28:12.958 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:12 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:28:12.958 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:12 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:28:12.958 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:12 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:28:12.959 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:12 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:28:12.959 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:12 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-20T22:28:13.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:12 smithi194 ceph-mon[51725]: pgmap v237: 97 pgs: 9 active+undersized, 23 active+undersized+degraded, 65 active+clean; 333 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.0 MiB/s wr, 414 op/s; 168/1470 objects degraded (11.429%) 2024-09-20T22:28:13.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:12 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:28:13.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:12 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:28:13.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:12 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:28:13.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:12 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:28:13.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:12 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:28:13.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:12 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:28:13.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:12 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:28:13.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:12 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:28:13.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:12 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:28:13.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:12 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:28:13.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:12 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-20T22:28:13.879 INFO:teuthology.orchestra.run.smithi037.stdout: 5 245606 9.97 MB/sec execute 511 sec latency 67.766 ms 2024-09-20T22:28:13.941 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:13 smithi037 ceph-mon[65640]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-20T22:28:13.941 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:13 smithi037 ceph-mon[65640]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-20T22:28:14.094 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:28:13 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-5[58907]: 2024-09-20T22:28:13.836+0000 7f63ef256640 -1 osd.5 0 read_superblock omap replica is missing. 2024-09-20T22:28:14.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:13 smithi194 ceph-mon[51725]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-20T22:28:14.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:13 smithi194 ceph-mon[51725]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-20T22:28:14.879 INFO:teuthology.orchestra.run.smithi037.stdout: 5 245914 9.98 MB/sec execute 512 sec latency 53.826 ms 2024-09-20T22:28:14.999 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:14 smithi037 ceph-mon[65640]: pgmap v238: 97 pgs: 9 active+undersized, 23 active+undersized+degraded, 65 active+clean; 335 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.8 MiB/s wr, 454 op/s; 184/1578 objects degraded (11.660%) 2024-09-20T22:28:14.999 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:14 smithi037 ceph-mon[65640]: Health check update: Degraded data redundancy: 184/1578 objects degraded (11.660%), 23 pgs degraded (PG_DEGRADED) 2024-09-20T22:28:15.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:14 smithi194 ceph-mon[51725]: pgmap v238: 97 pgs: 9 active+undersized, 23 active+undersized+degraded, 65 active+clean; 335 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.8 MiB/s wr, 454 op/s; 184/1578 objects degraded (11.660%) 2024-09-20T22:28:15.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:14 smithi194 ceph-mon[51725]: Health check update: Degraded data redundancy: 184/1578 objects degraded (11.660%), 23 pgs degraded (PG_DEGRADED) 2024-09-20T22:28:15.345 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:28:14 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-5[58907]: 2024-09-20T22:28:14.953+0000 7f63ef256640 -1 osd.5 130 log_to_monitors true 2024-09-20T22:28:15.345 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:28:15 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-5[58907]: 2024-09-20T22:28:15.027+0000 7f63e7001640 -1 osd.5 130 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-20T22:28:15.879 INFO:teuthology.orchestra.run.smithi037.stdout: 5 246094 9.97 MB/sec execute 513 sec latency 101.525 ms 2024-09-20T22:28:16.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:15 smithi037 ceph-mon[65640]: from='osd.5 [v2:172.21.15.194:6808/2044499583,v1:172.21.15.194:6809/2044499583]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-20T22:28:16.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:15 smithi037 ceph-mon[65640]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-20T22:28:16.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:15 smithi037 ceph-mon[65640]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-20T22:28:16.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:15 smithi037 ceph-mon[65640]: from='osd.5 [v2:172.21.15.194:6808/2044499583,v1:172.21.15.194:6809/2044499583]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2024-09-20T22:28:16.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:15 smithi037 ceph-mon[65640]: osdmap e133: 8 total, 7 up, 8 in 2024-09-20T22:28:16.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:15 smithi037 ceph-mon[65640]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2024-09-20T22:28:16.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:15 smithi194 ceph-mon[51725]: from='osd.5 [v2:172.21.15.194:6808/2044499583,v1:172.21.15.194:6809/2044499583]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-20T22:28:16.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:15 smithi194 ceph-mon[51725]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-20T22:28:16.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:15 smithi194 ceph-mon[51725]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-20T22:28:16.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:15 smithi194 ceph-mon[51725]: from='osd.5 [v2:172.21.15.194:6808/2044499583,v1:172.21.15.194:6809/2044499583]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2024-09-20T22:28:16.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:15 smithi194 ceph-mon[51725]: osdmap e133: 8 total, 7 up, 8 in 2024-09-20T22:28:16.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:15 smithi194 ceph-mon[51725]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2024-09-20T22:28:16.879 INFO:teuthology.orchestra.run.smithi037.stdout: 5 246147 9.95 MB/sec execute 514 sec latency 870.154 ms 2024-09-20T22:28:17.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:16 smithi194 ceph-mon[51725]: pgmap v240: 97 pgs: 9 active+undersized, 23 active+undersized+degraded, 65 active+clean; 335 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 3.4 MiB/s wr, 318 op/s; 180/1572 objects degraded (11.450%) 2024-09-20T22:28:17.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:16 smithi194 ceph-mon[51725]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-20T22:28:17.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:16 smithi194 ceph-mon[51725]: osd.5 [v2:172.21.15.194:6808/2044499583,v1:172.21.15.194:6809/2044499583] boot 2024-09-20T22:28:17.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:16 smithi194 ceph-mon[51725]: osdmap e134: 8 total, 8 up, 8 in 2024-09-20T22:28:17.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:16 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-20T22:28:17.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:16 smithi037 ceph-mon[65640]: pgmap v240: 97 pgs: 9 active+undersized, 23 active+undersized+degraded, 65 active+clean; 335 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 3.4 MiB/s wr, 318 op/s; 180/1572 objects degraded (11.450%) 2024-09-20T22:28:17.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:16 smithi037 ceph-mon[65640]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-20T22:28:17.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:16 smithi037 ceph-mon[65640]: osd.5 [v2:172.21.15.194:6808/2044499583,v1:172.21.15.194:6809/2044499583] boot 2024-09-20T22:28:17.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:16 smithi037 ceph-mon[65640]: osdmap e134: 8 total, 8 up, 8 in 2024-09-20T22:28:17.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:16 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-20T22:28:17.879 INFO:teuthology.orchestra.run.smithi037.stdout: 5 246147 9.93 MB/sec execute 515 sec latency 1870.204 ms 2024-09-20T22:28:18.276 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:18 smithi037 ceph-mon[65640]: osdmap e135: 8 total, 8 up, 8 in 2024-09-20T22:28:18.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:18 smithi194 ceph-mon[51725]: osdmap e135: 8 total, 8 up, 8 in 2024-09-20T22:28:18.879 INFO:teuthology.orchestra.run.smithi037.stdout: 5 246297 9.91 MB/sec execute 516 sec latency 2045.081 ms 2024-09-20T22:28:19.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:19 smithi037 ceph-mon[65640]: pgmap v243: 97 pgs: 1 active+recovering+degraded, 1 remapped+peering, 7 peering, 1 active+recovery_wait+degraded, 5 active+undersized, 13 active+undersized+degraded, 69 active+clean; 336 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.2 MiB/s wr, 474 op/s; 124/1617 objects degraded (7.669%); 15 KiB/s, 4 objects/s recovering 2024-09-20T22:28:19.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:19 smithi037 ceph-mon[65640]: osdmap e136: 8 total, 8 up, 8 in 2024-09-20T22:28:19.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:19 smithi194 ceph-mon[51725]: pgmap v243: 97 pgs: 1 active+recovering+degraded, 1 remapped+peering, 7 peering, 1 active+recovery_wait+degraded, 5 active+undersized, 13 active+undersized+degraded, 69 active+clean; 336 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.2 MiB/s wr, 474 op/s; 124/1617 objects degraded (7.669%); 15 KiB/s, 4 objects/s recovering 2024-09-20T22:28:19.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:19 smithi194 ceph-mon[51725]: osdmap e136: 8 total, 8 up, 8 in 2024-09-20T22:28:19.879 INFO:teuthology.orchestra.run.smithi037.stdout: 5 246418 9.90 MB/sec execute 517 sec latency 576.045 ms 2024-09-20T22:28:20.064 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:20 smithi037 ceph-mon[65640]: osdmap e137: 8 total, 8 up, 8 in 2024-09-20T22:28:20.064 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:20 smithi037 ceph-mon[65640]: Health check update: Degraded data redundancy: 124/1617 objects degraded (7.669%), 15 pgs degraded (PG_DEGRADED) 2024-09-20T22:28:20.064 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:20 smithi037 ceph-mon[65640]: osdmap e138: 8 total, 8 up, 8 in 2024-09-20T22:28:20.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:20 smithi194 ceph-mon[51725]: osdmap e137: 8 total, 8 up, 8 in 2024-09-20T22:28:20.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:20 smithi194 ceph-mon[51725]: Health check update: Degraded data redundancy: 124/1617 objects degraded (7.669%), 15 pgs degraded (PG_DEGRADED) 2024-09-20T22:28:20.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:20 smithi194 ceph-mon[51725]: osdmap e138: 8 total, 8 up, 8 in 2024-09-20T22:28:20.879 INFO:teuthology.orchestra.run.smithi037.stdout: 5 246625 9.88 MB/sec execute 518 sec latency 699.155 ms 2024-09-20T22:28:21.247 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:21 smithi037 ceph-mon[65640]: pgmap v247: 97 pgs: 1 active+recovering+degraded, 1 remapped+peering, 7 peering, 1 active+recovery_wait+degraded, 4 active+undersized, 11 active+undersized+degraded, 72 active+clean; 337 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.1 MiB/s wr, 437 op/s; 99/1629 objects degraded (6.077%); 101 KiB/s, 17 objects/s recovering 2024-09-20T22:28:21.247 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:21 smithi037 ceph-mon[65640]: osdmap e139: 8 total, 8 up, 8 in 2024-09-20T22:28:21.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:21 smithi194 ceph-mon[51725]: pgmap v247: 97 pgs: 1 active+recovering+degraded, 1 remapped+peering, 7 peering, 1 active+recovery_wait+degraded, 4 active+undersized, 11 active+undersized+degraded, 72 active+clean; 337 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.1 MiB/s wr, 437 op/s; 99/1629 objects degraded (6.077%); 101 KiB/s, 17 objects/s recovering 2024-09-20T22:28:21.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:21 smithi194 ceph-mon[51725]: osdmap e139: 8 total, 8 up, 8 in 2024-09-20T22:28:21.880 INFO:teuthology.orchestra.run.smithi037.stdout: 5 246855 9.87 MB/sec execute 519 sec latency 77.444 ms 2024-09-20T22:28:22.880 INFO:teuthology.orchestra.run.smithi037.stdout: 5 247110 9.87 MB/sec execute 520 sec latency 40.453 ms 2024-09-20T22:28:23.042 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:22 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:28:23.042 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:22 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:28:23.042 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:22 smithi037 ceph-mon[65640]: pgmap v249: 97 pgs: 1 active+recovering+degraded, 1 remapped+peering, 4 peering, 1 active+recovery_wait+degraded, 90 active+clean; 339 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 1.1 MiB/s wr, 60 op/s; 13/1650 objects degraded (0.788%); 1.6 MiB/s, 34 objects/s recovering 2024-09-20T22:28:23.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:22 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:28:23.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:22 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:28:23.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:22 smithi194 ceph-mon[51725]: pgmap v249: 97 pgs: 1 active+recovering+degraded, 1 remapped+peering, 4 peering, 1 active+recovery_wait+degraded, 90 active+clean; 339 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 1.1 MiB/s wr, 60 op/s; 13/1650 objects degraded (0.788%); 1.6 MiB/s, 34 objects/s recovering 2024-09-20T22:28:23.880 INFO:teuthology.orchestra.run.smithi037.stdout: 5 247429 9.87 MB/sec execute 521 sec latency 64.632 ms 2024-09-20T22:28:24.053 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:24 smithi037 ceph-mon[65640]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 13/1650 objects degraded (0.788%), 2 pgs degraded) 2024-09-20T22:28:24.054 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:24 smithi037 ceph-mon[65640]: Cluster is now healthy 2024-09-20T22:28:24.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:24 smithi194 ceph-mon[51725]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 13/1650 objects degraded (0.788%), 2 pgs degraded) 2024-09-20T22:28:24.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:24 smithi194 ceph-mon[51725]: Cluster is now healthy 2024-09-20T22:28:24.880 INFO:teuthology.orchestra.run.smithi037.stdout: 5 247794 9.86 MB/sec execute 522 sec latency 77.797 ms 2024-09-20T22:28:25.066 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:25 smithi037 ceph-mon[65640]: pgmap v250: 97 pgs: 2 peering, 95 active+clean; 337 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 1.8 MiB/s wr, 126 op/s; 1.7 MiB/s, 29 objects/s recovering 2024-09-20T22:28:25.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:25 smithi194 ceph-mon[51725]: pgmap v250: 97 pgs: 2 peering, 95 active+clean; 337 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 1.8 MiB/s wr, 126 op/s; 1.7 MiB/s, 29 objects/s recovering 2024-09-20T22:28:25.880 INFO:teuthology.orchestra.run.smithi037.stdout: 5 248110 9.85 MB/sec execute 523 sec latency 40.512 ms 2024-09-20T22:28:26.880 INFO:teuthology.orchestra.run.smithi037.stdout: 5 248268 9.84 MB/sec execute 524 sec latency 56.434 ms 2024-09-20T22:28:27.128 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:27 smithi037 ceph-mon[65640]: pgmap v251: 97 pgs: 97 active+clean; 334 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.2 KiB/s rd, 2.8 MiB/s wr, 339 op/s; 1.5 MiB/s, 30 objects/s recovering 2024-09-20T22:28:27.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:27 smithi194 ceph-mon[51725]: pgmap v251: 97 pgs: 97 active+clean; 334 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.2 KiB/s rd, 2.8 MiB/s wr, 339 op/s; 1.5 MiB/s, 30 objects/s recovering 2024-09-20T22:28:27.880 INFO:teuthology.orchestra.run.smithi037.stdout: 5 248598 9.85 MB/sec execute 525 sec latency 59.549 ms 2024-09-20T22:28:28.104 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:28 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-20T22:28:28.135 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:28 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-20T22:28:28.880 INFO:teuthology.orchestra.run.smithi037.stdout: 5 248897 9.85 MB/sec execute 526 sec latency 66.924 ms 2024-09-20T22:28:29.119 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:29 smithi194 ceph-mon[51725]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-20T22:28:29.120 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:29 smithi194 ceph-mon[51725]: Upgrade: osd.6 is safe to restart 2024-09-20T22:28:29.120 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:29 smithi194 ceph-mon[51725]: pgmap v252: 97 pgs: 97 active+clean; 331 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 3.4 MiB/s wr, 343 op/s; 1.2 MiB/s, 25 objects/s recovering 2024-09-20T22:28:29.175 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:29 smithi037 ceph-mon[65640]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-20T22:28:29.176 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:29 smithi037 ceph-mon[65640]: Upgrade: osd.6 is safe to restart 2024-09-20T22:28:29.176 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:29 smithi037 ceph-mon[65640]: pgmap v252: 97 pgs: 97 active+clean; 331 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 3.4 MiB/s wr, 343 op/s; 1.2 MiB/s, 25 objects/s recovering 2024-09-20T22:28:29.880 INFO:teuthology.orchestra.run.smithi037.stdout: 5 249242 9.86 MB/sec execute 527 sec latency 51.621 ms 2024-09-20T22:28:30.180 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:30 smithi037 ceph-mon[65640]: Upgrade: Updating osd.6 2024-09-20T22:28:30.180 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:30 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:28:30.180 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:30 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-20T22:28:30.180 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:30 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:28:30.180 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:30 smithi037 ceph-mon[65640]: Deploying daemon osd.6 on smithi194 2024-09-20T22:28:30.593 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:30 smithi194 ceph-mon[51725]: Upgrade: Updating osd.6 2024-09-20T22:28:30.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:30 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:28:30.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:30 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-20T22:28:30.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:30 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:28:30.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:30 smithi194 ceph-mon[51725]: Deploying daemon osd.6 on smithi194 2024-09-20T22:28:30.880 INFO:teuthology.orchestra.run.smithi037.stdout: 5 249481 9.86 MB/sec execute 528 sec latency 78.744 ms 2024-09-20T22:28:31.191 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:31 smithi037 ceph-mon[65640]: pgmap v253: 97 pgs: 97 active+clean; 333 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 3.2 MiB/s wr, 299 op/s; 1013 KiB/s, 16 objects/s recovering 2024-09-20T22:28:31.499 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:31 smithi194 ceph-mon[51725]: pgmap v253: 97 pgs: 97 active+clean; 333 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 3.2 MiB/s wr, 299 op/s; 1013 KiB/s, 16 objects/s recovering 2024-09-20T22:28:31.880 INFO:teuthology.orchestra.run.smithi037.stdout: 5 249762 9.85 MB/sec execute 529 sec latency 78.802 ms 2024-09-20T22:28:32.845 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:28:32 smithi194 systemd[1]: Stopping Ceph osd.6 for f9a6357a-779c-11ef-baf5-efdc52797490... 2024-09-20T22:28:32.880 INFO:teuthology.orchestra.run.smithi037.stdout: 5 250105 9.83 MB/sec execute 530 sec latency 51.031 ms 2024-09-20T22:28:33.233 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:32 smithi037 ceph-mon[65640]: pgmap v254: 97 pgs: 97 active+clean; 336 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.1 MiB/s wr, 337 op/s; 775 KiB/s, 8 objects/s recovering 2024-09-20T22:28:33.233 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:32 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:28:33.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:32 smithi194 ceph-mon[51725]: pgmap v254: 97 pgs: 97 active+clean; 336 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.1 MiB/s wr, 337 op/s; 775 KiB/s, 8 objects/s recovering 2024-09-20T22:28:33.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:32 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:28:33.345 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:28:33 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6[36079]: 2024-09-20T22:28:33.192+0000 7f68da3f4700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-20T22:28:33.345 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:28:33 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6[36079]: 2024-09-20T22:28:33.192+0000 7f68da3f4700 -1 osd.6 139 *** Got signal Terminated *** 2024-09-20T22:28:33.345 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:28:33 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6[36079]: 2024-09-20T22:28:33.192+0000 7f68da3f4700 -1 osd.6 139 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-20T22:28:33.880 INFO:teuthology.orchestra.run.smithi037.stdout: 5 250384 9.83 MB/sec execute 531 sec latency 39.069 ms 2024-09-20T22:28:33.986 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:33 smithi037 ceph-mon[65640]: osd.6 marked itself down and dead 2024-09-20T22:28:34.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:33 smithi194 ceph-mon[51725]: osd.6 marked itself down and dead 2024-09-20T22:28:34.880 INFO:teuthology.orchestra.run.smithi037.stdout: 5 250471 9.82 MB/sec execute 532 sec latency 874.514 ms 2024-09-20T22:28:34.981 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:28:34 smithi194 podman[60676]: 2024-09-20 22:28:34.628407387 +0000 UTC m=+1.662283368 container died 27f0720639ed449ba7141c4b40fa1e7886ac6a3355aaf8150d6824e18a969e01 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., ceph=True, distribution-scope=public, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, RELEASE=HEAD, release=754, vcs-type=git, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, name=centos-stream, version=8) 2024-09-20T22:28:35.233 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:34 smithi037 ceph-mon[65640]: pgmap v255: 97 pgs: 97 active+clean; 341 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.3 MiB/s wr, 436 op/s; 336 KiB/s, 5 objects/s recovering 2024-09-20T22:28:35.233 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:34 smithi037 ceph-mon[65640]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-20T22:28:35.233 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:34 smithi037 ceph-mon[65640]: osdmap e140: 8 total, 7 up, 8 in 2024-09-20T22:28:35.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:34 smithi194 ceph-mon[51725]: pgmap v255: 97 pgs: 97 active+clean; 341 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.3 MiB/s wr, 436 op/s; 336 KiB/s, 5 objects/s recovering 2024-09-20T22:28:35.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:34 smithi194 ceph-mon[51725]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-20T22:28:35.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:34 smithi194 ceph-mon[51725]: osdmap e140: 8 total, 7 up, 8 in 2024-09-20T22:28:35.345 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:28:35 smithi194 podman[60676]: 2024-09-20 22:28:35.044232345 +0000 UTC m=+2.078108318 container cleanup 27f0720639ed449ba7141c4b40fa1e7886ac6a3355aaf8150d6824e18a969e01 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6, vcs-type=git, name=centos-stream, architecture=x86_64, RELEASE=HEAD, vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, io.openshift.expose-services=, version=8) 2024-09-20T22:28:35.345 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:28:35 smithi194 bash[60676]: ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6 2024-09-20T22:28:35.844 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:28:35 smithi194 podman[60725]: 2024-09-20 22:28:35.413635146 +0000 UTC m=+0.789002094 container remove 27f0720639ed449ba7141c4b40fa1e7886ac6a3355aaf8150d6824e18a969e01 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6, GIT_BRANCH=HEAD, architecture=x86_64, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , name=centos-stream, version=8, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8) 2024-09-20T22:28:35.880 INFO:teuthology.orchestra.run.smithi037.stdout: 5 250778 9.82 MB/sec execute 533 sec latency 1012.396 ms 2024-09-20T22:28:35.996 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:35 smithi037 ceph-mon[65640]: osdmap e141: 8 total, 7 up, 8 in 2024-09-20T22:28:36.205 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:35 smithi194 ceph-mon[51725]: osdmap e141: 8 total, 7 up, 8 in 2024-09-20T22:28:36.205 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:28:35 smithi194 podman[60810]: 2024-09-20 22:28:35.856474696 +0000 UTC m=+0.018776752 image pull cc9251b744a3faafca7c3095f79bfff81873afe823f31a03a11e015a34abcdbd quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5 2024-09-20T22:28:36.206 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:28:35 smithi194 podman[60810]: 2024-09-20 22:28:35.964582618 +0000 UTC m=+0.126884663 container create 292bf0131cca9dd881837bab18e5f60d7aca9b4b6b1b83319cee9f2e171137c6 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6-deactivate, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=squid-9e2ac70, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-09-20T22:28:36.206 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:28:36 smithi194 podman[60810]: 2024-09-20 22:28:36.200812575 +0000 UTC m=+0.363114636 container init 292bf0131cca9dd881837bab18e5f60d7aca9b4b6b1b83319cee9f2e171137c6 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6-deactivate, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-9e2ac70, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-09-20T22:28:36.206 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:28:36 smithi194 podman[60810]: 2024-09-20 22:28:36.204439087 +0000 UTC m=+0.366741136 container start 292bf0131cca9dd881837bab18e5f60d7aca9b4b6b1b83319cee9f2e171137c6 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6-deactivate, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, RELEASE=squid-9e2ac70, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, ceph=True) 2024-09-20T22:28:36.594 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:28:36 smithi194 podman[60810]: 2024-09-20 22:28:36.300971956 +0000 UTC m=+0.463274019 container attach 292bf0131cca9dd881837bab18e5f60d7aca9b4b6b1b83319cee9f2e171137c6 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6-deactivate, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=squid-9e2ac70, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, ceph=True) 2024-09-20T22:28:36.595 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:28:36 smithi194 podman[60810]: 2024-09-20 22:28:36.430718693 +0000 UTC m=+0.593020742 container died 292bf0131cca9dd881837bab18e5f60d7aca9b4b6b1b83319cee9f2e171137c6 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6-deactivate, RELEASE=squid-9e2ac70, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731) 2024-09-20T22:28:36.880 INFO:teuthology.orchestra.run.smithi037.stdout: 5 251138 9.82 MB/sec execute 534 sec latency 67.063 ms 2024-09-20T22:28:37.017 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:36 smithi037 ceph-mon[65640]: pgmap v258: 97 pgs: 2 active+undersized, 1 peering, 7 stale+active+clean, 5 active+undersized+degraded, 82 active+clean; 341 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 4.0 MiB/s wr, 350 op/s; 38/1644 objects degraded (2.311%) 2024-09-20T22:28:37.017 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:36 smithi037 ceph-mon[65640]: Health check failed: Degraded data redundancy: 38/1644 objects degraded (2.311%), 5 pgs degraded (PG_DEGRADED) 2024-09-20T22:28:37.017 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:36 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:28:37.017 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:36 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:28:37.271 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:36 smithi194 ceph-mon[51725]: pgmap v258: 97 pgs: 2 active+undersized, 1 peering, 7 stale+active+clean, 5 active+undersized+degraded, 82 active+clean; 341 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 4.0 MiB/s wr, 350 op/s; 38/1644 objects degraded (2.311%) 2024-09-20T22:28:37.271 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:36 smithi194 ceph-mon[51725]: Health check failed: Degraded data redundancy: 38/1644 objects degraded (2.311%), 5 pgs degraded (PG_DEGRADED) 2024-09-20T22:28:37.271 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:36 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:28:37.271 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:36 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:28:37.271 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:28:37 smithi194 podman[60833]: 2024-09-20 22:28:37.030630368 +0000 UTC m=+0.593085751 container remove 292bf0131cca9dd881837bab18e5f60d7aca9b4b6b1b83319cee9f2e171137c6 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6-deactivate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-9e2ac70) 2024-09-20T22:28:37.271 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:28:37 smithi194 systemd[1]: ceph-f9a6357a-779c-11ef-baf5-efdc52797490@osd.6.service: Deactivated successfully. 2024-09-20T22:28:37.272 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:28:37 smithi194 systemd[1]: Stopped Ceph osd.6 for f9a6357a-779c-11ef-baf5-efdc52797490. 2024-09-20T22:28:37.272 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:28:37 smithi194 systemd[1]: ceph-f9a6357a-779c-11ef-baf5-efdc52797490@osd.6.service: Consumed 4min 18.710s CPU time. 2024-09-20T22:28:37.525 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:28:37 smithi194 systemd[1]: Starting Ceph osd.6 for f9a6357a-779c-11ef-baf5-efdc52797490... 2024-09-20T22:28:37.525 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:28:37 smithi194 podman[60930]: 2024-09-20 22:28:37.397145312 +0000 UTC m=+0.019045680 image pull cc9251b744a3faafca7c3095f79bfff81873afe823f31a03a11e015a34abcdbd quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5 2024-09-20T22:28:37.525 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:28:37 smithi194 podman[60930]: 2024-09-20 22:28:37.523882578 +0000 UTC m=+0.145782926 container create 252ff77e5d917ba5bc2072ab88bb1dd68d7a29f1986a13abd16ca09cf751fd42 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6-activate, maintainer=Guillaume Abrioux , RELEASE=squid-9e2ac70, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-20T22:28:37.881 INFO:teuthology.orchestra.run.smithi037.stdout: 5 251533 9.81 MB/sec execute 535 sec latency 51.743 ms 2024-09-20T22:28:38.095 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:28:37 smithi194 podman[60930]: 2024-09-20 22:28:37.793694722 +0000 UTC m=+0.415595071 container init 252ff77e5d917ba5bc2072ab88bb1dd68d7a29f1986a13abd16ca09cf751fd42 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6-activate, org.label-schema.license=GPLv2, RELEASE=squid-9e2ac70, ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.37.2) 2024-09-20T22:28:38.095 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:28:37 smithi194 podman[60930]: 2024-09-20 22:28:37.798334544 +0000 UTC m=+0.420234895 container start 252ff77e5d917ba5bc2072ab88bb1dd68d7a29f1986a13abd16ca09cf751fd42 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6-activate, RELEASE=squid-9e2ac70, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True) 2024-09-20T22:28:38.095 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:28:37 smithi194 podman[60930]: 2024-09-20 22:28:37.901894874 +0000 UTC m=+0.523795234 container attach 252ff77e5d917ba5bc2072ab88bb1dd68d7a29f1986a13abd16ca09cf751fd42 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6-activate, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, GIT_CLEAN=True, io.buildah.version=1.37.2, RELEASE=squid-9e2ac70, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-20T22:28:38.095 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:28:37 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6-activate[60944]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T22:28:38.095 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:28:37 smithi194 bash[60930]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T22:28:38.095 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:28:37 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6-activate[60944]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T22:28:38.096 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:28:37 smithi194 bash[60930]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T22:28:38.880 INFO:teuthology.orchestra.run.smithi037.stdout: 5 251764 9.80 MB/sec execute 536 sec latency 43.325 ms 2024-09-20T22:28:38.999 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:28:38 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6-activate[60944]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-20T22:28:38.999 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:28:38 smithi194 bash[60930]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-20T22:28:38.999 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:28:38 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6-activate[60944]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-20T22:28:38.999 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:28:38 smithi194 bash[60930]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-20T22:28:39.078 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:38 smithi037 ceph-mon[65640]: pgmap v259: 97 pgs: 10 active+undersized, 1 peering, 24 active+undersized+degraded, 62 active+clean; 338 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 5.0 MiB/s wr, 519 op/s; 168/1557 objects degraded (10.790%) 2024-09-20T22:28:39.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:38 smithi194 ceph-mon[51725]: pgmap v259: 97 pgs: 10 active+undersized, 1 peering, 24 active+undersized+degraded, 62 active+clean; 338 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 5.0 MiB/s wr, 519 op/s; 168/1557 objects degraded (10.790%) 2024-09-20T22:28:39.345 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:28:39 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6-activate[60944]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-20T22:28:39.345 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:28:39 smithi194 bash[60930]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-20T22:28:39.345 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:28:39 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6-activate[60944]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-20T22:28:39.345 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:28:39 smithi194 bash[60930]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-20T22:28:39.345 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:28:39 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6-activate[60944]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-09-20T22:28:39.345 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:28:39 smithi194 bash[60930]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-09-20T22:28:39.345 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:28:39 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6-activate[60944]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-20T22:28:39.345 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:28:39 smithi194 bash[60930]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-20T22:28:39.345 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:28:39 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6-activate[60944]: --> ceph-volume raw activate successful for osd ID: 6 2024-09-20T22:28:39.345 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:28:39 smithi194 bash[60930]: --> ceph-volume raw activate successful for osd ID: 6 2024-09-20T22:28:39.345 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:28:39 smithi194 podman[60930]: 2024-09-20 22:28:39.0659285 +0000 UTC m=+1.687828845 container died 252ff77e5d917ba5bc2072ab88bb1dd68d7a29f1986a13abd16ca09cf751fd42 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6-activate, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=squid-9e2ac70, GIT_CLEAN=True, org.label-schema.license=GPLv2, io.buildah.version=1.37.2) 2024-09-20T22:28:39.880 INFO:teuthology.orchestra.run.smithi037.stdout: 5 252012 9.80 MB/sec execute 537 sec latency 64.976 ms 2024-09-20T22:28:40.216 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:28:39 smithi194 podman[61069]: 2024-09-20 22:28:39.865374519 +0000 UTC m=+0.788083009 container remove 252ff77e5d917ba5bc2072ab88bb1dd68d7a29f1986a13abd16ca09cf751fd42 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6-activate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=squid-9e2ac70, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-09-20T22:28:40.216 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:28:40 smithi194 podman[61124]: 2024-09-20 22:28:39.979863318 +0000 UTC m=+0.016432303 image pull cc9251b744a3faafca7c3095f79bfff81873afe823f31a03a11e015a34abcdbd quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5 2024-09-20T22:28:40.216 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:28:40 smithi194 podman[61124]: 2024-09-20 22:28:40.091325525 +0000 UTC m=+0.127894497 container create 2daa9faae9a2ce6f545c58f6a5647a58ee50ef154591aaa2abfb315907bb4e40 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, RELEASE=squid-9e2ac70, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-09-20T22:28:40.646 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:40 smithi037 ceph-mon[65640]: pgmap v260: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 338 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 4.5 MiB/s wr, 413 op/s; 181/1575 objects degraded (11.492%) 2024-09-20T22:28:40.646 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:40 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:28:40.646 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:40 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:28:40.646 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:40 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:28:40.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:40 smithi194 ceph-mon[51725]: pgmap v260: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 338 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 4.5 MiB/s wr, 413 op/s; 181/1575 objects degraded (11.492%) 2024-09-20T22:28:40.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:40 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:28:40.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:40 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:28:40.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:40 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:28:40.845 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:28:40 smithi194 podman[61124]: 2024-09-20 22:28:40.344304153 +0000 UTC m=+0.380873135 container init 2daa9faae9a2ce6f545c58f6a5647a58ee50ef154591aaa2abfb315907bb4e40 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=squid-9e2ac70, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-09-20T22:28:40.845 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:28:40 smithi194 podman[61124]: 2024-09-20 22:28:40.348020618 +0000 UTC m=+0.384589591 container start 2daa9faae9a2ce6f545c58f6a5647a58ee50ef154591aaa2abfb315907bb4e40 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, RELEASE=squid-9e2ac70) 2024-09-20T22:28:40.845 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:28:40 smithi194 ceph-osd[61142]: -- 172.21.15.194:0/1529233284 <== mon.1 v2:172.21.15.194:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55a7a11461e0 con 0x55a7a050c000 2024-09-20T22:28:40.845 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:28:40 smithi194 bash[61124]: 2daa9faae9a2ce6f545c58f6a5647a58ee50ef154591aaa2abfb315907bb4e40 2024-09-20T22:28:40.845 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:28:40 smithi194 systemd[1]: Started Ceph osd.6 for f9a6357a-779c-11ef-baf5-efdc52797490. 2024-09-20T22:28:40.880 INFO:teuthology.orchestra.run.smithi037.stdout: 5 252359 9.80 MB/sec execute 538 sec latency 61.952 ms 2024-09-20T22:28:41.747 INFO:teuthology.orchestra.run.smithi037.stdout:true 2024-09-20T22:28:41.880 INFO:teuthology.orchestra.run.smithi037.stdout: 5 252744 9.81 MB/sec execute 539 sec latency 53.657 ms 2024-09-20T22:28:42.167 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:42 smithi037 ceph-mon[65640]: Health check update: Degraded data redundancy: 178/1578 objects degraded (11.280%), 25 pgs degraded (PG_DEGRADED) 2024-09-20T22:28:42.285 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:42 smithi194 ceph-mon[51725]: Health check update: Degraded data redundancy: 178/1578 objects degraded (11.280%), 25 pgs degraded (PG_DEGRADED) 2024-09-20T22:28:42.285 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:28:42 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6[61138]: 2024-09-20T22:28:42.219+0000 7f5957a6e640 -1 Falling back to public interface 2024-09-20T22:28:42.709 INFO:teuthology.orchestra.run.smithi037.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T22:28:42.709 INFO:teuthology.orchestra.run.smithi037.stdout:alertmanager.smithi037 smithi037 *:9093,9094 running (8m) 110s ago 17m 25.4M - 0.25.0 c8568f914cd2 d1f99b3cd707 2024-09-20T22:28:42.709 INFO:teuthology.orchestra.run.smithi037.stdout:crash.smithi037 smithi037 running (5m) 110s ago 17m 6895k - 19.1.1-367-g9e2ac70b cc9251b744a3 77e510d8bdb1 2024-09-20T22:28:42.709 INFO:teuthology.orchestra.run.smithi037.stdout:crash.smithi194 smithi194 running (5m) 34s ago 16m 6895k - 19.1.1-367-g9e2ac70b cc9251b744a3 4d169d1309bc 2024-09-20T22:28:42.709 INFO:teuthology.orchestra.run.smithi037.stdout:grafana.smithi037 smithi037 *:3000 running (8m) 110s ago 17m 53.2M - dad864ee21e9 2306f3c2da19 2024-09-20T22:28:42.709 INFO:teuthology.orchestra.run.smithi037.stdout:mds.foofs.smithi037.sdtedi smithi037 running (11m) 110s ago 11m 37.9M - 17.2.0 e1d6a67b021e 0a7898d25b8d 2024-09-20T22:28:42.709 INFO:teuthology.orchestra.run.smithi037.stdout:mds.foofs.smithi194.lkroso smithi194 running (11m) 34s ago 11m 16.5M - 17.2.0 e1d6a67b021e 17736de30008 2024-09-20T22:28:42.710 INFO:teuthology.orchestra.run.smithi037.stdout:mgr.smithi037.lykblh smithi037 *:8443,9283,8765 running (7m) 110s ago 18m 505M - 19.1.1-367-g9e2ac70b cc9251b744a3 181e46fd9ba6 2024-09-20T22:28:42.710 INFO:teuthology.orchestra.run.smithi037.stdout:mgr.smithi194.bkcglt smithi194 *:8443,9283,8765 running (6m) 34s ago 16m 443M - 19.1.1-367-g9e2ac70b cc9251b744a3 1fa610c97517 2024-09-20T22:28:42.710 INFO:teuthology.orchestra.run.smithi037.stdout:mon.smithi037 smithi037 running (6m) 110s ago 18m 52.0M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 c0229eceb527 2024-09-20T22:28:42.710 INFO:teuthology.orchestra.run.smithi037.stdout:mon.smithi194 smithi194 running (6m) 34s ago 16m 46.1M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 97854807ba91 2024-09-20T22:28:42.710 INFO:teuthology.orchestra.run.smithi037.stdout:nfs.foo.0.0.smithi037.spabdc smithi037 *:2049 running (11m) 110s ago 11m 280M - 4.0 e1d6a67b021e 858ff9d558a2 2024-09-20T22:28:42.710 INFO:teuthology.orchestra.run.smithi037.stdout:nfs.foo.1.0.smithi194.qzjyej smithi194 *:2049 running (11m) 34s ago 11m 90.3M - 4.0 e1d6a67b021e c07fe0eb7c75 2024-09-20T22:28:42.710 INFO:teuthology.orchestra.run.smithi037.stdout:node-exporter.smithi037 smithi037 *:9100 running (8m) 110s ago 17m 15.4M - 1.5.0 0da6a335fe13 433e6aaf6137 2024-09-20T22:28:42.710 INFO:teuthology.orchestra.run.smithi037.stdout:node-exporter.smithi194 smithi194 *:9100 running (7m) 34s ago 16m 15.8M - 1.5.0 0da6a335fe13 1d30bb810ad1 2024-09-20T22:28:42.710 INFO:teuthology.orchestra.run.smithi037.stdout:osd.0 smithi037 running (4m) 110s ago 15m 367M 1937M 19.1.1-367-g9e2ac70b cc9251b744a3 c5568bd43c07 2024-09-20T22:28:42.710 INFO:teuthology.orchestra.run.smithi037.stdout:osd.1 smithi037 running (3m) 110s ago 15m 269M 1937M 19.1.1-367-g9e2ac70b cc9251b744a3 f6b715391469 2024-09-20T22:28:42.710 INFO:teuthology.orchestra.run.smithi037.stdout:osd.2 smithi037 running (2m) 110s ago 14m 211M 1937M 19.1.1-367-g9e2ac70b cc9251b744a3 0922768db2a2 2024-09-20T22:28:42.711 INFO:teuthology.orchestra.run.smithi037.stdout:osd.3 smithi037 running (119s) 110s ago 14m 12.4M 1937M 19.1.1-367-g9e2ac70b cc9251b744a3 312635406c2a 2024-09-20T22:28:42.711 INFO:teuthology.orchestra.run.smithi037.stdout:osd.4 smithi194 running (76s) 34s ago 14m 324M 2704M 19.1.1-367-g9e2ac70b cc9251b744a3 641138d63e50 2024-09-20T22:28:42.711 INFO:teuthology.orchestra.run.smithi037.stdout:osd.5 smithi194 running (39s) 34s ago 13m 13.7M 2704M 19.1.1-367-g9e2ac70b cc9251b744a3 9a2a44a8db21 2024-09-20T22:28:42.711 INFO:teuthology.orchestra.run.smithi037.stdout:osd.6 smithi194 starting - - - 2704M 2024-09-20T22:28:42.711 INFO:teuthology.orchestra.run.smithi037.stdout:osd.7 smithi194 running (13m) 34s ago 13m 793M 2704M 17.2.0 e1d6a67b021e 62d163f4fd92 2024-09-20T22:28:42.711 INFO:teuthology.orchestra.run.smithi037.stdout:prometheus.smithi037 smithi037 *:9095 running (6m) 110s ago 16m 66.0M - 2.43.0 a07b618ecd1d 05b826dde2ff 2024-09-20T22:28:42.880 INFO:teuthology.orchestra.run.smithi037.stdout: 5 253040 9.82 MB/sec execute 540 sec latency 48.308 ms 2024-09-20T22:28:43.196 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:43 smithi037 ceph-mon[65640]: from='client.34404 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:28:43.196 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:43 smithi037 ceph-mon[65640]: pgmap v261: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 342 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 3.9 KiB/s rd, 3.3 MiB/s wr, 409 op/s; 178/1578 objects degraded (11.280%) 2024-09-20T22:28:43.263 INFO:teuthology.orchestra.run.smithi037.stdout:{ 2024-09-20T22:28:43.263 INFO:teuthology.orchestra.run.smithi037.stdout: "mon": { 2024-09-20T22:28:43.263 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 2 2024-09-20T22:28:43.263 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:28:43.263 INFO:teuthology.orchestra.run.smithi037.stdout: "mgr": { 2024-09-20T22:28:43.263 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 2 2024-09-20T22:28:43.263 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:28:43.263 INFO:teuthology.orchestra.run.smithi037.stdout: "osd": { 2024-09-20T22:28:43.263 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-20T22:28:43.263 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 6 2024-09-20T22:28:43.263 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:28:43.263 INFO:teuthology.orchestra.run.smithi037.stdout: "mds": { 2024-09-20T22:28:43.264 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T22:28:43.264 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:28:43.264 INFO:teuthology.orchestra.run.smithi037.stdout: "overall": { 2024-09-20T22:28:43.264 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-09-20T22:28:43.264 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 10 2024-09-20T22:28:43.264 INFO:teuthology.orchestra.run.smithi037.stdout: } 2024-09-20T22:28:43.264 INFO:teuthology.orchestra.run.smithi037.stdout:} 2024-09-20T22:28:43.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:43 smithi194 ceph-mon[51725]: from='client.34404 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:28:43.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:43 smithi194 ceph-mon[51725]: pgmap v261: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 342 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 3.9 KiB/s rd, 3.3 MiB/s wr, 409 op/s; 178/1578 objects degraded (11.280%) 2024-09-20T22:28:43.764 INFO:teuthology.orchestra.run.smithi037.stdout:{ 2024-09-20T22:28:43.764 INFO:teuthology.orchestra.run.smithi037.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5", 2024-09-20T22:28:43.764 INFO:teuthology.orchestra.run.smithi037.stdout: "in_progress": true, 2024-09-20T22:28:43.764 INFO:teuthology.orchestra.run.smithi037.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-20T22:28:43.765 INFO:teuthology.orchestra.run.smithi037.stdout: "services_complete": [ 2024-09-20T22:28:43.765 INFO:teuthology.orchestra.run.smithi037.stdout: "mon", 2024-09-20T22:28:43.765 INFO:teuthology.orchestra.run.smithi037.stdout: "mgr", 2024-09-20T22:28:43.765 INFO:teuthology.orchestra.run.smithi037.stdout: "crash" 2024-09-20T22:28:43.765 INFO:teuthology.orchestra.run.smithi037.stdout: ], 2024-09-20T22:28:43.765 INFO:teuthology.orchestra.run.smithi037.stdout: "progress": "12/23 daemons upgraded", 2024-09-20T22:28:43.765 INFO:teuthology.orchestra.run.smithi037.stdout: "message": "Currently upgrading osd daemons", 2024-09-20T22:28:43.765 INFO:teuthology.orchestra.run.smithi037.stdout: "is_paused": false 2024-09-20T22:28:43.765 INFO:teuthology.orchestra.run.smithi037.stdout:} 2024-09-20T22:28:43.880 INFO:teuthology.orchestra.run.smithi037.stdout: 5 253276 9.81 MB/sec execute 541 sec latency 54.344 ms 2024-09-20T22:28:44.214 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:44 smithi037 ceph-mon[65640]: from='client.34408 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:28:44.214 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:44 smithi037 ceph-mon[65640]: from='client.34412 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:28:44.214 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:44 smithi037 ceph-mon[65640]: from='client.? 172.21.15.37:0/2942222491' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:28:44.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:44 smithi194 ceph-mon[51725]: from='client.34408 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:28:44.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:44 smithi194 ceph-mon[51725]: from='client.34412 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:28:44.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:44 smithi194 ceph-mon[51725]: from='client.? 172.21.15.37:0/2942222491' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:28:44.415 INFO:teuthology.orchestra.run.smithi037.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 196/1638 objects degraded (11.966%), 25 pgs degraded 2024-09-20T22:28:44.415 INFO:teuthology.orchestra.run.smithi037.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-20T22:28:44.415 INFO:teuthology.orchestra.run.smithi037.stdout: osd.6 (root=default,host=smithi194) is down 2024-09-20T22:28:44.415 INFO:teuthology.orchestra.run.smithi037.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 196/1638 objects degraded (11.966%), 25 pgs degraded 2024-09-20T22:28:44.415 INFO:teuthology.orchestra.run.smithi037.stdout: pg 1.0 is active+undersized+degraded, acting [7,0] 2024-09-20T22:28:44.415 INFO:teuthology.orchestra.run.smithi037.stdout: pg 2.2 is active+undersized+degraded, acting [5,1] 2024-09-20T22:28:44.415 INFO:teuthology.orchestra.run.smithi037.stdout: pg 2.6 is active+undersized+degraded, acting [1,4] 2024-09-20T22:28:44.415 INFO:teuthology.orchestra.run.smithi037.stdout: pg 2.7 is active+undersized+degraded, acting [4,2] 2024-09-20T22:28:44.415 INFO:teuthology.orchestra.run.smithi037.stdout: pg 2.11 is active+undersized+degraded, acting [4,1] 2024-09-20T22:28:44.416 INFO:teuthology.orchestra.run.smithi037.stdout: pg 2.14 is active+undersized+degraded, acting [3,5] 2024-09-20T22:28:44.416 INFO:teuthology.orchestra.run.smithi037.stdout: pg 2.16 is active+undersized+degraded, acting [5,2] 2024-09-20T22:28:44.416 INFO:teuthology.orchestra.run.smithi037.stdout: pg 2.17 is active+undersized+degraded, acting [5,2] 2024-09-20T22:28:44.416 INFO:teuthology.orchestra.run.smithi037.stdout: pg 2.19 is active+undersized+degraded, acting [3,0] 2024-09-20T22:28:44.416 INFO:teuthology.orchestra.run.smithi037.stdout: pg 2.1a is active+undersized+degraded, acting [4,7] 2024-09-20T22:28:44.416 INFO:teuthology.orchestra.run.smithi037.stdout: pg 2.1b is active+undersized+degraded, acting [3,7] 2024-09-20T22:28:44.416 INFO:teuthology.orchestra.run.smithi037.stdout: pg 2.1d is active+undersized+degraded, acting [7,0] 2024-09-20T22:28:44.416 INFO:teuthology.orchestra.run.smithi037.stdout: pg 3.0 is active+undersized+degraded, acting [1,2] 2024-09-20T22:28:44.416 INFO:teuthology.orchestra.run.smithi037.stdout: pg 3.2 is active+undersized+degraded, acting [3,5] 2024-09-20T22:28:44.416 INFO:teuthology.orchestra.run.smithi037.stdout: pg 3.3 is active+undersized+degraded, acting [4,0] 2024-09-20T22:28:44.416 INFO:teuthology.orchestra.run.smithi037.stdout: pg 3.a is active+undersized+degraded, acting [4,1] 2024-09-20T22:28:44.416 INFO:teuthology.orchestra.run.smithi037.stdout: pg 3.c is active+undersized+degraded, acting [5,3] 2024-09-20T22:28:44.416 INFO:teuthology.orchestra.run.smithi037.stdout: pg 3.d is active+undersized+degraded, acting [7,5] 2024-09-20T22:28:44.416 INFO:teuthology.orchestra.run.smithi037.stdout: pg 3.10 is active+undersized+degraded, acting [0,5] 2024-09-20T22:28:44.417 INFO:teuthology.orchestra.run.smithi037.stdout: pg 3.11 is active+undersized+degraded, acting [7,4] 2024-09-20T22:28:44.417 INFO:teuthology.orchestra.run.smithi037.stdout: pg 3.14 is active+undersized+degraded, acting [4,7] 2024-09-20T22:28:44.417 INFO:teuthology.orchestra.run.smithi037.stdout: pg 3.1d is active+undersized+degraded, acting [5,4] 2024-09-20T22:28:44.417 INFO:teuthology.orchestra.run.smithi037.stdout: pg 3.1e is active+undersized+degraded, acting [3,2] 2024-09-20T22:28:44.417 INFO:teuthology.orchestra.run.smithi037.stdout: pg 4.5 is active+undersized+degraded, acting [4,7] 2024-09-20T22:28:44.417 INFO:teuthology.orchestra.run.smithi037.stdout: pg 4.a is active+undersized+degraded, acting [1,7] 2024-09-20T22:28:44.880 INFO:teuthology.orchestra.run.smithi037.stdout: 5 253578 9.79 MB/sec execute 542 sec latency 47.528 ms 2024-09-20T22:28:45.261 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:45 smithi037 ceph-mon[65640]: from='client.34420 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:28:45.261 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:45 smithi037 ceph-mon[65640]: pgmap v262: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 345 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 3.1 KiB/s rd, 4.2 MiB/s wr, 434 op/s; 196/1638 objects degraded (11.966%) 2024-09-20T22:28:45.261 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:45 smithi037 ceph-mon[65640]: from='client.? 172.21.15.37:0/2396252216' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T22:28:45.261 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:45 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:28:45.261 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:45 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:28:45.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:45 smithi194 ceph-mon[51725]: from='client.34420 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:28:45.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:45 smithi194 ceph-mon[51725]: pgmap v262: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 345 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 3.1 KiB/s rd, 4.2 MiB/s wr, 434 op/s; 196/1638 objects degraded (11.966%) 2024-09-20T22:28:45.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:45 smithi194 ceph-mon[51725]: from='client.? 172.21.15.37:0/2396252216' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T22:28:45.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:45 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:28:45.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:45 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:28:45.880 INFO:teuthology.orchestra.run.smithi037.stdout: 5 253793 9.78 MB/sec execute 543 sec latency 71.204 ms 2024-09-20T22:28:46.294 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:46 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:28:46.294 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:46 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:28:46.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:46 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:28:46.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:46 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:28:46.880 INFO:teuthology.orchestra.run.smithi037.stdout: 5 254059 9.78 MB/sec execute 544 sec latency 68.033 ms 2024-09-20T22:28:47.315 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:47 smithi037 ceph-mon[65640]: pgmap v263: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 345 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.3 MiB/s wr, 443 op/s; 205/1698 objects degraded (12.073%) 2024-09-20T22:28:47.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:47 smithi194 ceph-mon[51725]: pgmap v263: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 345 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.3 MiB/s wr, 443 op/s; 205/1698 objects degraded (12.073%) 2024-09-20T22:28:47.880 INFO:teuthology.orchestra.run.smithi037.stdout: 5 254499 9.79 MB/sec execute 545 sec latency 64.720 ms 2024-09-20T22:28:48.362 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:48 smithi037 ceph-mon[65640]: pgmap v264: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 344 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.3 KiB/s rd, 4.8 MiB/s wr, 482 op/s; 178/1584 objects degraded (11.237%) 2024-09-20T22:28:48.362 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:48 smithi037 ceph-mon[65640]: Health check update: Degraded data redundancy: 178/1584 objects degraded (11.237%), 25 pgs degraded (PG_DEGRADED) 2024-09-20T22:28:48.586 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:48 smithi194 ceph-mon[51725]: pgmap v264: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 344 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.3 KiB/s rd, 4.8 MiB/s wr, 482 op/s; 178/1584 objects degraded (11.237%) 2024-09-20T22:28:48.586 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:48 smithi194 ceph-mon[51725]: Health check update: Degraded data redundancy: 178/1584 objects degraded (11.237%), 25 pgs degraded (PG_DEGRADED) 2024-09-20T22:28:48.881 INFO:teuthology.orchestra.run.smithi037.stdout: 5 254973 9.78 MB/sec execute 546 sec latency 52.842 ms 2024-09-20T22:28:49.881 INFO:teuthology.orchestra.run.smithi037.stdout: 5 255281 9.78 MB/sec execute 547 sec latency 70.440 ms 2024-09-20T22:28:50.157 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:49 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:28:50.157 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:49 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:28:50.157 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:49 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:28:50.157 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:49 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:28:50.158 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:49 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:28:50.158 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:49 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:28:50.158 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:49 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:28:50.158 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:49 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:28:50.158 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:49 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:28:50.158 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:49 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:28:50.158 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:49 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-20T22:28:50.158 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:49 smithi037 ceph-mon[65640]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-20T22:28:50.158 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:49 smithi037 ceph-mon[65640]: Upgrade: unsafe to stop osd(s) at this time (11 PGs are or would become offline) 2024-09-20T22:28:50.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:49 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:28:50.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:49 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:28:50.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:49 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:28:50.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:49 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:28:50.346 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:49 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:28:50.346 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:49 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:28:50.346 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:49 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:28:50.346 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:49 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:28:50.346 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:49 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:28:50.347 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:49 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:28:50.347 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:49 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-20T22:28:50.347 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:49 smithi194 ceph-mon[51725]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-20T22:28:50.347 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:49 smithi194 ceph-mon[51725]: Upgrade: unsafe to stop osd(s) at this time (11 PGs are or would become offline) 2024-09-20T22:28:50.844 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:28:50 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6[61138]: 2024-09-20T22:28:50.486+0000 7f5957a6e640 -1 osd.6 0 read_superblock omap replica is missing. 2024-09-20T22:28:50.881 INFO:teuthology.orchestra.run.smithi037.stdout: 5 255573 9.77 MB/sec execute 548 sec latency 56.345 ms 2024-09-20T22:28:51.173 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:50 smithi037 ceph-mon[65640]: pgmap v265: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 343 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.3 KiB/s rd, 4.3 MiB/s wr, 423 op/s; 174/1548 objects degraded (11.240%) 2024-09-20T22:28:51.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:50 smithi194 ceph-mon[51725]: pgmap v265: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 343 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.3 KiB/s rd, 4.3 MiB/s wr, 423 op/s; 174/1548 objects degraded (11.240%) 2024-09-20T22:28:51.881 INFO:teuthology.orchestra.run.smithi037.stdout: 5 255913 9.78 MB/sec execute 549 sec latency 58.766 ms 2024-09-20T22:28:52.029 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:28:51 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6[61138]: 2024-09-20T22:28:51.736+0000 7f5957a6e640 -1 osd.6 139 log_to_monitors true 2024-09-20T22:28:52.227 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:52 smithi037 ceph-mon[65640]: from='osd.6 [v2:172.21.15.194:6816/2054132986,v1:172.21.15.194:6817/2054132986]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-20T22:28:52.227 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:52 smithi037 ceph-mon[65640]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-20T22:28:52.227 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:52 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:28:52.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:52 smithi194 ceph-mon[51725]: from='osd.6 [v2:172.21.15.194:6816/2054132986,v1:172.21.15.194:6817/2054132986]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-20T22:28:52.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:52 smithi194 ceph-mon[51725]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-20T22:28:52.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:52 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:28:52.345 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:28:52 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6[61138]: 2024-09-20T22:28:52.056+0000 7f594f819640 -1 osd.6 139 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-20T22:28:52.881 INFO:teuthology.orchestra.run.smithi037.stdout: 5 256261 9.79 MB/sec execute 550 sec latency 63.902 ms 2024-09-20T22:28:53.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:53 smithi194 ceph-mon[51725]: pgmap v266: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 339 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.3 MiB/s wr, 430 op/s; 173/1530 objects degraded (11.307%) 2024-09-20T22:28:53.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:53 smithi194 ceph-mon[51725]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-20T22:28:53.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:53 smithi194 ceph-mon[51725]: osdmap e142: 8 total, 7 up, 8 in 2024-09-20T22:28:53.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:53 smithi194 ceph-mon[51725]: from='osd.6 [v2:172.21.15.194:6816/2054132986,v1:172.21.15.194:6817/2054132986]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2024-09-20T22:28:53.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:53 smithi194 ceph-mon[51725]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2024-09-20T22:28:53.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:53 smithi037 ceph-mon[65640]: pgmap v266: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 339 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.3 MiB/s wr, 430 op/s; 173/1530 objects degraded (11.307%) 2024-09-20T22:28:53.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:53 smithi037 ceph-mon[65640]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-20T22:28:53.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:53 smithi037 ceph-mon[65640]: osdmap e142: 8 total, 7 up, 8 in 2024-09-20T22:28:53.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:53 smithi037 ceph-mon[65640]: from='osd.6 [v2:172.21.15.194:6816/2054132986,v1:172.21.15.194:6817/2054132986]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2024-09-20T22:28:53.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:53 smithi037 ceph-mon[65640]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2024-09-20T22:28:53.881 INFO:teuthology.orchestra.run.smithi037.stdout: 5 256335 9.77 MB/sec execute 551 sec latency 787.756 ms 2024-09-20T22:28:54.288 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:54 smithi037 ceph-mon[65640]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-20T22:28:54.289 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:54 smithi037 ceph-mon[65640]: osd.6 [v2:172.21.15.194:6816/2054132986,v1:172.21.15.194:6817/2054132986] boot 2024-09-20T22:28:54.289 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:54 smithi037 ceph-mon[65640]: osdmap e143: 8 total, 8 up, 8 in 2024-09-20T22:28:54.289 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:54 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-20T22:28:54.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:54 smithi194 ceph-mon[51725]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-20T22:28:54.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:54 smithi194 ceph-mon[51725]: osd.6 [v2:172.21.15.194:6816/2054132986,v1:172.21.15.194:6817/2054132986] boot 2024-09-20T22:28:54.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:54 smithi194 ceph-mon[51725]: osdmap e143: 8 total, 8 up, 8 in 2024-09-20T22:28:54.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:54 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-20T22:28:54.881 INFO:teuthology.orchestra.run.smithi037.stdout: 5 256530 9.77 MB/sec execute 552 sec latency 1787.794 ms 2024-09-20T22:28:55.052 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:55 smithi037 ceph-mon[65640]: pgmap v269: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 344 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.1 KiB/s rd, 4.5 MiB/s wr, 452 op/s; 180/1575 objects degraded (11.429%) 2024-09-20T22:28:55.052 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:55 smithi037 ceph-mon[65640]: Health check update: Degraded data redundancy: 180/1575 objects degraded (11.429%), 25 pgs degraded (PG_DEGRADED) 2024-09-20T22:28:55.052 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:55 smithi037 ceph-mon[65640]: osdmap e144: 8 total, 8 up, 8 in 2024-09-20T22:28:55.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:55 smithi194 ceph-mon[51725]: pgmap v269: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 344 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.1 KiB/s rd, 4.5 MiB/s wr, 452 op/s; 180/1575 objects degraded (11.429%) 2024-09-20T22:28:55.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:55 smithi194 ceph-mon[51725]: Health check update: Degraded data redundancy: 180/1575 objects degraded (11.429%), 25 pgs degraded (PG_DEGRADED) 2024-09-20T22:28:55.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:55 smithi194 ceph-mon[51725]: osdmap e144: 8 total, 8 up, 8 in 2024-09-20T22:28:55.881 INFO:teuthology.orchestra.run.smithi037.stdout: 5 256767 9.77 MB/sec execute 553 sec latency 2016.854 ms 2024-09-20T22:28:56.077 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:56 smithi037 ceph-mon[65640]: osdmap e145: 8 total, 8 up, 8 in 2024-09-20T22:28:56.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:56 smithi194 ceph-mon[51725]: osdmap e145: 8 total, 8 up, 8 in 2024-09-20T22:28:56.881 INFO:teuthology.orchestra.run.smithi037.stdout: 5 256983 9.75 MB/sec execute 554 sec latency 71.178 ms 2024-09-20T22:28:57.087 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:57 smithi037 ceph-mon[65640]: pgmap v272: 97 pgs: 1 remapped+peering, 9 active+undersized, 21 active+undersized+degraded, 66 active+clean; 347 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.4 MiB/s wr, 691 op/s; 141/1614 objects degraded (8.736%); 36 KiB/s, 5 objects/s recovering 2024-09-20T22:28:57.087 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:57 smithi037 ceph-mon[65640]: osdmap e146: 8 total, 8 up, 8 in 2024-09-20T22:28:57.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:57 smithi194 ceph-mon[51725]: pgmap v272: 97 pgs: 1 remapped+peering, 9 active+undersized, 21 active+undersized+degraded, 66 active+clean; 347 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.4 MiB/s wr, 691 op/s; 141/1614 objects degraded (8.736%); 36 KiB/s, 5 objects/s recovering 2024-09-20T22:28:57.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:57 smithi194 ceph-mon[51725]: osdmap e146: 8 total, 8 up, 8 in 2024-09-20T22:28:57.881 INFO:teuthology.orchestra.run.smithi037.stdout: 5 257267 9.74 MB/sec execute 555 sec latency 642.272 ms 2024-09-20T22:28:58.116 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:58 smithi037 ceph-mon[65640]: osdmap e147: 8 total, 8 up, 8 in 2024-09-20T22:28:58.116 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:58 smithi037 ceph-mon[65640]: osdmap e148: 8 total, 8 up, 8 in 2024-09-20T22:28:58.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:58 smithi194 ceph-mon[51725]: osdmap e147: 8 total, 8 up, 8 in 2024-09-20T22:28:58.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:58 smithi194 ceph-mon[51725]: osdmap e148: 8 total, 8 up, 8 in 2024-09-20T22:28:58.881 INFO:teuthology.orchestra.run.smithi037.stdout: 5 257590 9.74 MB/sec execute 556 sec latency 867.925 ms 2024-09-20T22:28:59.133 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:59 smithi037 ceph-mon[65640]: pgmap v275: 97 pgs: 1 active+undersized+remapped, 2 peering, 1 remapped+peering, 93 active+clean; 344 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.0 KiB/s rd, 4.4 MiB/s wr, 383 op/s; 769 KiB/s, 66 keys/s, 54 objects/s recovering 2024-09-20T22:28:59.133 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:59 smithi037 ceph-mon[65640]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 141/1614 objects degraded (8.736%), 21 pgs degraded) 2024-09-20T22:28:59.133 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:28:59 smithi037 ceph-mon[65640]: Cluster is now healthy 2024-09-20T22:28:59.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:59 smithi194 ceph-mon[51725]: pgmap v275: 97 pgs: 1 active+undersized+remapped, 2 peering, 1 remapped+peering, 93 active+clean; 344 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.0 KiB/s rd, 4.4 MiB/s wr, 383 op/s; 769 KiB/s, 66 keys/s, 54 objects/s recovering 2024-09-20T22:28:59.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:59 smithi194 ceph-mon[51725]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 141/1614 objects degraded (8.736%), 21 pgs degraded) 2024-09-20T22:28:59.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:28:59 smithi194 ceph-mon[51725]: Cluster is now healthy 2024-09-20T22:28:59.881 INFO:teuthology.orchestra.run.smithi037.stdout: 5 258043 9.75 MB/sec execute 557 sec latency 42.739 ms 2024-09-20T22:29:00.882 INFO:teuthology.orchestra.run.smithi037.stdout: 5 258531 9.74 MB/sec execute 558 sec latency 68.232 ms 2024-09-20T22:29:01.266 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:01 smithi037 ceph-mon[65640]: pgmap v277: 97 pgs: 1 active+undersized+remapped, 2 peering, 1 remapped+peering, 93 active+clean; 343 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.6 KiB/s rd, 2.1 MiB/s wr, 100 op/s; 605 KiB/s, 55 keys/s, 40 objects/s recovering 2024-09-20T22:29:01.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:01 smithi194 ceph-mon[51725]: pgmap v277: 97 pgs: 1 active+undersized+remapped, 2 peering, 1 remapped+peering, 93 active+clean; 343 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.6 KiB/s rd, 2.1 MiB/s wr, 100 op/s; 605 KiB/s, 55 keys/s, 40 objects/s recovering 2024-09-20T22:29:01.881 INFO:teuthology.orchestra.run.smithi037.stdout: 5 258946 9.74 MB/sec execute 559 sec latency 44.090 ms 2024-09-20T22:29:02.881 INFO:teuthology.orchestra.run.smithi037.stdout: 5 259327 9.75 MB/sec execute 560 sec latency 51.988 ms 2024-09-20T22:29:03.299 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:03 smithi037 ceph-mon[65640]: pgmap v278: 97 pgs: 1 active+undersized+remapped, 2 peering, 94 active+clean; 341 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.3 KiB/s rd, 2.9 MiB/s wr, 275 op/s; 504 KiB/s, 44 keys/s, 36 objects/s recovering 2024-09-20T22:29:03.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:03 smithi194 ceph-mon[51725]: pgmap v278: 97 pgs: 1 active+undersized+remapped, 2 peering, 94 active+clean; 341 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.3 KiB/s rd, 2.9 MiB/s wr, 275 op/s; 504 KiB/s, 44 keys/s, 36 objects/s recovering 2024-09-20T22:29:03.881 INFO:teuthology.orchestra.run.smithi037.stdout: 5 259784 9.77 MB/sec execute 561 sec latency 46.682 ms 2024-09-20T22:29:04.881 INFO:teuthology.orchestra.run.smithi037.stdout: 5 260200 9.77 MB/sec execute 562 sec latency 48.473 ms 2024-09-20T22:29:05.322 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:05 smithi037 ceph-mon[65640]: pgmap v279: 97 pgs: 97 active+clean; 338 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 9.8 KiB/s rd, 4.2 MiB/s wr, 446 op/s; 908 KiB/s, 34 keys/s, 28 objects/s recovering 2024-09-20T22:29:05.322 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:05 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-20T22:29:05.322 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:05 smithi037 ceph-mon[65640]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-20T22:29:05.322 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:05 smithi037 ceph-mon[65640]: Upgrade: osd.7 is safe to restart 2024-09-20T22:29:05.574 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:05 smithi194 ceph-mon[51725]: pgmap v279: 97 pgs: 97 active+clean; 338 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 9.8 KiB/s rd, 4.2 MiB/s wr, 446 op/s; 908 KiB/s, 34 keys/s, 28 objects/s recovering 2024-09-20T22:29:05.574 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:05 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-20T22:29:05.575 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:05 smithi194 ceph-mon[51725]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-20T22:29:05.575 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:05 smithi194 ceph-mon[51725]: Upgrade: osd.7 is safe to restart 2024-09-20T22:29:05.881 INFO:teuthology.orchestra.run.smithi037.stdout: 5 260503 9.77 MB/sec execute 563 sec latency 30.622 ms 2024-09-20T22:29:06.839 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:06 smithi037 ceph-mon[65640]: Upgrade: Updating osd.7 2024-09-20T22:29:06.839 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:06 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:29:06.839 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:06 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-20T22:29:06.839 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:06 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:29:06.839 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:06 smithi037 ceph-mon[65640]: Deploying daemon osd.7 on smithi194 2024-09-20T22:29:06.839 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:06 smithi037 ceph-mon[65640]: pgmap v280: 97 pgs: 97 active+clean; 345 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 5.1 KiB/s rd, 4.9 MiB/s wr, 546 op/s; 495 KiB/s, 14 objects/s recovering 2024-09-20T22:29:06.876 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:06 smithi194 ceph-mon[51725]: Upgrade: Updating osd.7 2024-09-20T22:29:06.876 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:06 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:29:06.876 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:06 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-20T22:29:06.876 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:06 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:29:06.876 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:06 smithi194 ceph-mon[51725]: Deploying daemon osd.7 on smithi194 2024-09-20T22:29:06.877 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:06 smithi194 ceph-mon[51725]: pgmap v280: 97 pgs: 97 active+clean; 345 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 5.1 KiB/s rd, 4.9 MiB/s wr, 546 op/s; 495 KiB/s, 14 objects/s recovering 2024-09-20T22:29:06.881 INFO:teuthology.orchestra.run.smithi037.stdout: 5 260886 9.76 MB/sec execute 564 sec latency 60.128 ms 2024-09-20T22:29:07.881 INFO:teuthology.orchestra.run.smithi037.stdout: 5 261173 9.76 MB/sec execute 565 sec latency 71.220 ms 2024-09-20T22:29:08.109 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:07 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:29:08.110 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:07 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:29:08.153 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:07 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:29:08.153 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:07 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:29:08.881 INFO:teuthology.orchestra.run.smithi037.stdout: 5 261471 9.76 MB/sec execute 566 sec latency 63.748 ms 2024-09-20T22:29:09.134 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:08 smithi037 ceph-mon[65640]: pgmap v281: 97 pgs: 97 active+clean; 350 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 6.0 MiB/s wr, 627 op/s; 419 KiB/s, 2 objects/s recovering 2024-09-20T22:29:09.179 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:08 smithi194 ceph-mon[51725]: pgmap v281: 97 pgs: 97 active+clean; 350 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 6.0 MiB/s wr, 627 op/s; 419 KiB/s, 2 objects/s recovering 2024-09-20T22:29:09.844 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:29:09 smithi194 systemd[1]: Stopping Ceph osd.7 for f9a6357a-779c-11ef-baf5-efdc52797490... 2024-09-20T22:29:09.881 INFO:teuthology.orchestra.run.smithi037.stdout: 5 261776 9.75 MB/sec execute 567 sec latency 66.337 ms 2024-09-20T22:29:10.594 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:29:10 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[39126]: 2024-09-20T22:29:10.116+0000 7f6d53bd9700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-20T22:29:10.595 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:29:10 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[39126]: 2024-09-20T22:29:10.116+0000 7f6d53bd9700 -1 osd.7 148 *** Got signal Terminated *** 2024-09-20T22:29:10.595 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:29:10 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[39126]: 2024-09-20T22:29:10.116+0000 7f6d53bd9700 -1 osd.7 148 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-20T22:29:10.881 INFO:teuthology.orchestra.run.smithi037.stdout: 5 262116 9.74 MB/sec execute 568 sec latency 54.902 ms 2024-09-20T22:29:11.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:10 smithi194 ceph-mon[51725]: pgmap v282: 97 pgs: 97 active+clean; 344 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.4 MiB/s wr, 542 op/s; 353 KiB/s, 2 objects/s recovering 2024-09-20T22:29:11.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:10 smithi194 ceph-mon[51725]: osd.7 marked itself down and dead 2024-09-20T22:29:11.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:10 smithi037 ceph-mon[65640]: pgmap v282: 97 pgs: 97 active+clean; 344 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.4 MiB/s wr, 542 op/s; 353 KiB/s, 2 objects/s recovering 2024-09-20T22:29:11.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:10 smithi037 ceph-mon[65640]: osd.7 marked itself down and dead 2024-09-20T22:29:11.881 INFO:teuthology.orchestra.run.smithi037.stdout: 5 262159 9.73 MB/sec execute 569 sec latency 899.782 ms 2024-09-20T22:29:11.977 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:29:11 smithi194 podman[62917]: 2024-09-20 22:29:11.664214867 +0000 UTC m=+1.673003070 container died 62d163f4fd929155579d58a529d2ef08c513348d7cb8a9ff42117392ecddef41 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7, vcs-type=git, GIT_BRANCH=HEAD, architecture=x86_64, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, GIT_CLEAN=True, com.redhat.component=centos-stream-container, io.openshift.expose-services=, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, ceph=True, io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc.) 2024-09-20T22:29:12.227 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:11 smithi037 ceph-mon[65640]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-20T22:29:12.227 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:11 smithi037 ceph-mon[65640]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-09-20T22:29:12.227 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:11 smithi037 ceph-mon[65640]: osdmap e149: 8 total, 7 up, 8 in 2024-09-20T22:29:12.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:11 smithi194 ceph-mon[51725]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-20T22:29:12.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:11 smithi194 ceph-mon[51725]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-09-20T22:29:12.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:11 smithi194 ceph-mon[51725]: osdmap e149: 8 total, 7 up, 8 in 2024-09-20T22:29:12.345 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:29:12 smithi194 podman[62917]: 2024-09-20 22:29:12.174080014 +0000 UTC m=+2.182868219 container cleanup 62d163f4fd929155579d58a529d2ef08c513348d7cb8a9ff42117392ecddef41 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=-17.2.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, vcs-type=git, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, io.openshift.expose-services=, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, version=8, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, architecture=x86_64) 2024-09-20T22:29:12.345 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:29:12 smithi194 bash[62917]: ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7 2024-09-20T22:29:12.778 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:29:12 smithi194 podman[62932]: 2024-09-20 22:29:12.41769553 +0000 UTC m=+0.754425478 container remove 62d163f4fd929155579d58a529d2ef08c513348d7cb8a9ff42117392ecddef41 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, com.redhat.component=centos-stream-container, name=centos-stream, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, version=8, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-09-20T22:29:12.881 INFO:teuthology.orchestra.run.smithi037.stdout: 5 262421 9.71 MB/sec execute 570 sec latency 1040.748 ms 2024-09-20T22:29:13.001 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:12 smithi037 ceph-mon[65640]: pgmap v284: 97 pgs: 2 peering, 10 stale+active+clean, 85 active+clean; 341 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 5.9 MiB/s wr, 640 op/s; 410 KiB/s, 0 objects/s recovering 2024-09-20T22:29:13.001 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:12 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:29:13.001 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:12 smithi037 ceph-mon[65640]: osdmap e150: 8 total, 7 up, 8 in 2024-09-20T22:29:13.031 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:12 smithi194 ceph-mon[51725]: pgmap v284: 97 pgs: 2 peering, 10 stale+active+clean, 85 active+clean; 341 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 5.9 MiB/s wr, 640 op/s; 410 KiB/s, 0 objects/s recovering 2024-09-20T22:29:13.031 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:12 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:29:13.031 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:12 smithi194 ceph-mon[51725]: osdmap e150: 8 total, 7 up, 8 in 2024-09-20T22:29:13.032 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:29:12 smithi194 podman[63020]: 2024-09-20 22:29:12.680362755 +0000 UTC m=+0.017735049 image pull cc9251b744a3faafca7c3095f79bfff81873afe823f31a03a11e015a34abcdbd quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5 2024-09-20T22:29:13.032 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:29:12 smithi194 podman[63020]: 2024-09-20 22:29:12.802066097 +0000 UTC m=+0.139438405 container create 1a265ca0f5854d6258a866c67eb3fd004fe4d54163342185bd943dd8aef5e338 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7-deactivate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=squid-9e2ac70, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS) 2024-09-20T22:29:13.345 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:29:13 smithi194 podman[63020]: 2024-09-20 22:29:13.030144215 +0000 UTC m=+0.367516483 container init 1a265ca0f5854d6258a866c67eb3fd004fe4d54163342185bd943dd8aef5e338 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7-deactivate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=squid-9e2ac70, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2) 2024-09-20T22:29:13.345 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:29:13 smithi194 podman[63020]: 2024-09-20 22:29:13.034793007 +0000 UTC m=+0.372165271 container start 1a265ca0f5854d6258a866c67eb3fd004fe4d54163342185bd943dd8aef5e338 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7-deactivate, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=squid-9e2ac70, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-20T22:29:13.345 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:29:13 smithi194 podman[63020]: 2024-09-20 22:29:13.138642893 +0000 UTC m=+0.476015190 container attach 1a265ca0f5854d6258a866c67eb3fd004fe4d54163342185bd943dd8aef5e338 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7-deactivate, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=squid-9e2ac70) 2024-09-20T22:29:13.345 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:29:13 smithi194 podman[63020]: 2024-09-20 22:29:13.247349861 +0000 UTC m=+0.584722121 container died 1a265ca0f5854d6258a866c67eb3fd004fe4d54163342185bd943dd8aef5e338 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7-deactivate, org.label-schema.build-date=20240731, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=squid-9e2ac70, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-09-20T22:29:13.881 INFO:teuthology.orchestra.run.smithi037.stdout: 5 262629 9.71 MB/sec execute 571 sec latency 62.483 ms 2024-09-20T22:29:14.171 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:14 smithi194 ceph-mon[51725]: Health check failed: Degraded data redundancy: 92/1530 objects degraded (6.013%), 12 pgs degraded (PG_DEGRADED) 2024-09-20T22:29:14.171 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:29:13 smithi194 podman[63043]: 2024-09-20 22:29:13.918686 +0000 UTC m=+0.665299715 container remove 1a265ca0f5854d6258a866c67eb3fd004fe4d54163342185bd943dd8aef5e338 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7-deactivate, org.label-schema.schema-version=1.0, RELEASE=squid-9e2ac70, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-20T22:29:14.171 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:29:13 smithi194 systemd[1]: ceph-f9a6357a-779c-11ef-baf5-efdc52797490@osd.7.service: Deactivated successfully. 2024-09-20T22:29:14.171 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:29:13 smithi194 systemd[1]: Stopped Ceph osd.7 for f9a6357a-779c-11ef-baf5-efdc52797490. 2024-09-20T22:29:14.171 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:29:13 smithi194 systemd[1]: ceph-f9a6357a-779c-11ef-baf5-efdc52797490@osd.7.service: Consumed 6min 41.139s CPU time. 2024-09-20T22:29:14.171 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:29:14 smithi194 systemd[1]: Starting Ceph osd.7 for f9a6357a-779c-11ef-baf5-efdc52797490... 2024-09-20T22:29:14.284 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:14 smithi037 ceph-mon[65640]: Health check failed: Degraded data redundancy: 92/1530 objects degraded (6.013%), 12 pgs degraded (PG_DEGRADED) 2024-09-20T22:29:14.430 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:29:14 smithi194 podman[63144]: 2024-09-20 22:29:14.297578628 +0000 UTC m=+0.019558634 image pull cc9251b744a3faafca7c3095f79bfff81873afe823f31a03a11e015a34abcdbd quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5 2024-09-20T22:29:14.683 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:29:14 smithi194 podman[63144]: 2024-09-20 22:29:14.42876513 +0000 UTC m=+0.150745135 container create 4e15d39e716a6ce50a195774da51f0b531594390fa1893aa62a12e3b551ebb0b (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7-activate, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=squid-9e2ac70, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True) 2024-09-20T22:29:14.683 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:29:14 smithi194 podman[63144]: 2024-09-20 22:29:14.68194119 +0000 UTC m=+0.403921200 container init 4e15d39e716a6ce50a195774da51f0b531594390fa1893aa62a12e3b551ebb0b (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7-activate, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-9e2ac70, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-09-20T22:29:14.881 INFO:teuthology.orchestra.run.smithi037.stdout: 5 263039 9.72 MB/sec execute 572 sec latency 35.247 ms 2024-09-20T22:29:14.978 INFO:teuthology.orchestra.run.smithi037.stdout:true 2024-09-20T22:29:15.046 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:29:14 smithi194 podman[63144]: 2024-09-20 22:29:14.685505257 +0000 UTC m=+0.407485265 container start 4e15d39e716a6ce50a195774da51f0b531594390fa1893aa62a12e3b551ebb0b (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7-activate, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-9e2ac70, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-09-20T22:29:15.047 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:29:14 smithi194 podman[63144]: 2024-09-20 22:29:14.773706833 +0000 UTC m=+0.495686839 container attach 4e15d39e716a6ce50a195774da51f0b531594390fa1893aa62a12e3b551ebb0b (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7-activate, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, CEPH_POINT_RELEASE=, RELEASE=squid-9e2ac70, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD) 2024-09-20T22:29:15.047 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:29:14 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7-activate[63175]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T22:29:15.047 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:29:14 smithi194 bash[63144]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T22:29:15.047 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:29:14 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7-activate[63175]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T22:29:15.047 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:29:14 smithi194 bash[63144]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T22:29:15.300 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:15 smithi037 ceph-mon[65640]: pgmap v286: 97 pgs: 9 peering, 4 stale+active+clean, 12 active+undersized+degraded, 72 active+clean; 337 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 4.5 MiB/s wr, 438 op/s; 92/1530 objects degraded (6.013%) 2024-09-20T22:29:15.332 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:15 smithi194 ceph-mon[51725]: pgmap v286: 97 pgs: 9 peering, 4 stale+active+clean, 12 active+undersized+degraded, 72 active+clean; 337 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 4.5 MiB/s wr, 438 op/s; 92/1530 objects degraded (6.013%) 2024-09-20T22:29:15.595 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:29:15 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7-activate[63175]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-20T22:29:15.595 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:29:15 smithi194 bash[63144]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-20T22:29:15.595 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:29:15 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7-activate[63175]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-20T22:29:15.595 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:29:15 smithi194 bash[63144]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-20T22:29:15.881 INFO:teuthology.orchestra.run.smithi037.stdout: 5 263478 9.74 MB/sec execute 573 sec latency 61.080 ms 2024-09-20T22:29:15.963 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:29:15 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7-activate[63175]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-20T22:29:15.963 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:29:15 smithi194 bash[63144]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-20T22:29:15.963 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:29:15 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7-activate[63175]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-20T22:29:15.963 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:29:15 smithi194 bash[63144]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-20T22:29:15.963 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:29:15 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7-activate[63175]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-09-20T22:29:15.963 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:29:15 smithi194 bash[63144]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-09-20T22:29:15.963 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:29:15 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7-activate[63175]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-20T22:29:15.963 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:29:15 smithi194 bash[63144]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-20T22:29:15.963 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:29:15 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7-activate[63175]: --> ceph-volume raw activate successful for osd ID: 7 2024-09-20T22:29:15.963 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:29:15 smithi194 bash[63144]: --> ceph-volume raw activate successful for osd ID: 7 2024-09-20T22:29:15.964 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:29:15 smithi194 podman[63144]: 2024-09-20 22:29:15.658394512 +0000 UTC m=+1.380374517 container died 4e15d39e716a6ce50a195774da51f0b531594390fa1893aa62a12e3b551ebb0b (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7-activate, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=squid-9e2ac70, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, ceph=True) 2024-09-20T22:29:16.049 INFO:teuthology.orchestra.run.smithi037.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T22:29:16.049 INFO:teuthology.orchestra.run.smithi037.stdout:alertmanager.smithi037 smithi037 *:9093,9094 running (9m) 2m ago 18m 25.4M - 0.25.0 c8568f914cd2 d1f99b3cd707 2024-09-20T22:29:16.049 INFO:teuthology.orchestra.run.smithi037.stdout:crash.smithi037 smithi037 running (5m) 2m ago 17m 6895k - 19.1.1-367-g9e2ac70b cc9251b744a3 77e510d8bdb1 2024-09-20T22:29:16.049 INFO:teuthology.orchestra.run.smithi037.stdout:crash.smithi194 smithi194 running (5m) 31s ago 17m 6895k - 19.1.1-367-g9e2ac70b cc9251b744a3 4d169d1309bc 2024-09-20T22:29:16.049 INFO:teuthology.orchestra.run.smithi037.stdout:grafana.smithi037 smithi037 *:3000 running (9m) 2m ago 17m 53.2M - dad864ee21e9 2306f3c2da19 2024-09-20T22:29:16.049 INFO:teuthology.orchestra.run.smithi037.stdout:mds.foofs.smithi037.sdtedi smithi037 running (12m) 2m ago 12m 37.9M - 17.2.0 e1d6a67b021e 0a7898d25b8d 2024-09-20T22:29:16.049 INFO:teuthology.orchestra.run.smithi037.stdout:mds.foofs.smithi194.lkroso smithi194 running (12m) 31s ago 12m 16.6M - 17.2.0 e1d6a67b021e 17736de30008 2024-09-20T22:29:16.049 INFO:teuthology.orchestra.run.smithi037.stdout:mgr.smithi037.lykblh smithi037 *:8443,9283,8765 running (8m) 2m ago 19m 505M - 19.1.1-367-g9e2ac70b cc9251b744a3 181e46fd9ba6 2024-09-20T22:29:16.049 INFO:teuthology.orchestra.run.smithi037.stdout:mgr.smithi194.bkcglt smithi194 *:8443,9283,8765 running (7m) 31s ago 17m 443M - 19.1.1-367-g9e2ac70b cc9251b744a3 1fa610c97517 2024-09-20T22:29:16.049 INFO:teuthology.orchestra.run.smithi037.stdout:mon.smithi037 smithi037 running (7m) 2m ago 19m 52.0M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 c0229eceb527 2024-09-20T22:29:16.050 INFO:teuthology.orchestra.run.smithi037.stdout:mon.smithi194 smithi194 running (6m) 31s ago 17m 47.1M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 97854807ba91 2024-09-20T22:29:16.050 INFO:teuthology.orchestra.run.smithi037.stdout:nfs.foo.0.0.smithi037.spabdc smithi037 *:2049 running (11m) 2m ago 11m 280M - 4.0 e1d6a67b021e 858ff9d558a2 2024-09-20T22:29:16.050 INFO:teuthology.orchestra.run.smithi037.stdout:nfs.foo.1.0.smithi194.qzjyej smithi194 *:2049 running (11m) 31s ago 11m 90.3M - 4.0 e1d6a67b021e c07fe0eb7c75 2024-09-20T22:29:16.050 INFO:teuthology.orchestra.run.smithi037.stdout:node-exporter.smithi037 smithi037 *:9100 running (8m) 2m ago 17m 15.4M - 1.5.0 0da6a335fe13 433e6aaf6137 2024-09-20T22:29:16.050 INFO:teuthology.orchestra.run.smithi037.stdout:node-exporter.smithi194 smithi194 *:9100 running (8m) 31s ago 17m 16.2M - 1.5.0 0da6a335fe13 1d30bb810ad1 2024-09-20T22:29:16.050 INFO:teuthology.orchestra.run.smithi037.stdout:osd.0 smithi037 running (5m) 2m ago 16m 367M 1937M 19.1.1-367-g9e2ac70b cc9251b744a3 c5568bd43c07 2024-09-20T22:29:16.050 INFO:teuthology.orchestra.run.smithi037.stdout:osd.1 smithi037 running (4m) 2m ago 16m 269M 1937M 19.1.1-367-g9e2ac70b cc9251b744a3 f6b715391469 2024-09-20T22:29:16.050 INFO:teuthology.orchestra.run.smithi037.stdout:osd.2 smithi037 running (3m) 2m ago 15m 211M 1937M 19.1.1-367-g9e2ac70b cc9251b744a3 0922768db2a2 2024-09-20T22:29:16.050 INFO:teuthology.orchestra.run.smithi037.stdout:osd.3 smithi037 running (2m) 2m ago 15m 12.4M 1937M 19.1.1-367-g9e2ac70b cc9251b744a3 312635406c2a 2024-09-20T22:29:16.050 INFO:teuthology.orchestra.run.smithi037.stdout:osd.4 smithi194 running (109s) 31s ago 14m 355M 2704M 19.1.1-367-g9e2ac70b cc9251b744a3 641138d63e50 2024-09-20T22:29:16.050 INFO:teuthology.orchestra.run.smithi037.stdout:osd.5 smithi194 running (73s) 31s ago 14m 253M 2704M 19.1.1-367-g9e2ac70b cc9251b744a3 9a2a44a8db21 2024-09-20T22:29:16.050 INFO:teuthology.orchestra.run.smithi037.stdout:osd.6 smithi194 running (36s) 31s ago 13m 12.1M 2704M 19.1.1-367-g9e2ac70b cc9251b744a3 2daa9faae9a2 2024-09-20T22:29:16.050 INFO:teuthology.orchestra.run.smithi037.stdout:osd.7 smithi194 running (13m) 31s ago 13m 550M 2704M 17.2.0 e1d6a67b021e 62d163f4fd92 2024-09-20T22:29:16.050 INFO:teuthology.orchestra.run.smithi037.stdout:prometheus.smithi037 smithi037 *:9095 running (7m) 2m ago 16m 66.0M - 2.43.0 a07b618ecd1d 05b826dde2ff 2024-09-20T22:29:16.057 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:16 smithi037 ceph-mon[65640]: from='client.34428 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:29:16.280 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:16 smithi194 ceph-mon[51725]: from='client.34428 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:29:16.594 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:29:16 smithi194 podman[63331]: 2024-09-20 22:29:16.278847382 +0000 UTC m=+0.614344118 container remove 4e15d39e716a6ce50a195774da51f0b531594390fa1893aa62a12e3b551ebb0b (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7-activate, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=squid-9e2ac70, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2) 2024-09-20T22:29:16.595 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:29:16 smithi194 podman[63389]: 2024-09-20 22:29:16.394965616 +0000 UTC m=+0.017243740 image pull cc9251b744a3faafca7c3095f79bfff81873afe823f31a03a11e015a34abcdbd quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5 2024-09-20T22:29:16.595 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:29:16 smithi194 podman[63389]: 2024-09-20 22:29:16.513037267 +0000 UTC m=+0.135315378 container create 281a2861e7e639e0d27e5914a35b818214cd1c2bd53d5dcd8efa0a7b39dc2eed (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-9e2ac70, maintainer=Guillaume Abrioux ) 2024-09-20T22:29:16.721 INFO:teuthology.orchestra.run.smithi037.stdout:{ 2024-09-20T22:29:16.721 INFO:teuthology.orchestra.run.smithi037.stdout: "mon": { 2024-09-20T22:29:16.721 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 2 2024-09-20T22:29:16.722 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:29:16.722 INFO:teuthology.orchestra.run.smithi037.stdout: "mgr": { 2024-09-20T22:29:16.722 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 2 2024-09-20T22:29:16.722 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:29:16.722 INFO:teuthology.orchestra.run.smithi037.stdout: "osd": { 2024-09-20T22:29:16.722 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 7 2024-09-20T22:29:16.722 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:29:16.722 INFO:teuthology.orchestra.run.smithi037.stdout: "mds": { 2024-09-20T22:29:16.722 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T22:29:16.722 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:29:16.722 INFO:teuthology.orchestra.run.smithi037.stdout: "overall": { 2024-09-20T22:29:16.722 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-09-20T22:29:16.722 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 11 2024-09-20T22:29:16.722 INFO:teuthology.orchestra.run.smithi037.stdout: } 2024-09-20T22:29:16.723 INFO:teuthology.orchestra.run.smithi037.stdout:} 2024-09-20T22:29:16.881 INFO:teuthology.orchestra.run.smithi037.stdout: 5 263793 9.74 MB/sec execute 574 sec latency 47.952 ms 2024-09-20T22:29:17.055 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:29:16 smithi194 podman[63389]: 2024-09-20 22:29:16.765507112 +0000 UTC m=+0.387785232 container init 281a2861e7e639e0d27e5914a35b818214cd1c2bd53d5dcd8efa0a7b39dc2eed (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, RELEASE=squid-9e2ac70, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, ceph=True) 2024-09-20T22:29:17.055 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:29:16 smithi194 podman[63389]: 2024-09-20 22:29:16.769033611 +0000 UTC m=+0.391311727 container start 281a2861e7e639e0d27e5914a35b818214cd1c2bd53d5dcd8efa0a7b39dc2eed (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7, org.label-schema.vendor=CentOS, RELEASE=squid-9e2ac70, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240731) 2024-09-20T22:29:17.055 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:29:16 smithi194 ceph-osd[63408]: -- 172.21.15.194:0/2467925512 <== mon.1 v2:172.21.15.194:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x564dcee461e0 con 0x564dce037c00 2024-09-20T22:29:17.055 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:29:16 smithi194 bash[63389]: 281a2861e7e639e0d27e5914a35b818214cd1c2bd53d5dcd8efa0a7b39dc2eed 2024-09-20T22:29:17.055 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:29:16 smithi194 systemd[1]: Started Ceph osd.7 for f9a6357a-779c-11ef-baf5-efdc52797490. 2024-09-20T22:29:17.097 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:17 smithi037 ceph-mon[65640]: from='client.44353 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:29:17.097 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:17 smithi037 ceph-mon[65640]: pgmap v287: 97 pgs: 6 active+undersized, 9 peering, 22 active+undersized+degraded, 60 active+clean; 339 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 2.4 MiB/s wr, 269 op/s; 175/1533 objects degraded (11.416%) 2024-09-20T22:29:17.097 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:17 smithi037 ceph-mon[65640]: from='client.44357 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:29:17.097 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:17 smithi037 ceph-mon[65640]: from='client.? 172.21.15.37:0/2245079432' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:29:17.097 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:17 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:29:17.097 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:17 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:29:17.097 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:17 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:29:17.283 INFO:teuthology.orchestra.run.smithi037.stdout:{ 2024-09-20T22:29:17.283 INFO:teuthology.orchestra.run.smithi037.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5", 2024-09-20T22:29:17.283 INFO:teuthology.orchestra.run.smithi037.stdout: "in_progress": true, 2024-09-20T22:29:17.283 INFO:teuthology.orchestra.run.smithi037.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-20T22:29:17.283 INFO:teuthology.orchestra.run.smithi037.stdout: "services_complete": [ 2024-09-20T22:29:17.283 INFO:teuthology.orchestra.run.smithi037.stdout: "mon", 2024-09-20T22:29:17.283 INFO:teuthology.orchestra.run.smithi037.stdout: "mgr", 2024-09-20T22:29:17.283 INFO:teuthology.orchestra.run.smithi037.stdout: "crash" 2024-09-20T22:29:17.283 INFO:teuthology.orchestra.run.smithi037.stdout: ], 2024-09-20T22:29:17.283 INFO:teuthology.orchestra.run.smithi037.stdout: "progress": "13/23 daemons upgraded", 2024-09-20T22:29:17.284 INFO:teuthology.orchestra.run.smithi037.stdout: "message": "Currently upgrading osd daemons", 2024-09-20T22:29:17.284 INFO:teuthology.orchestra.run.smithi037.stdout: "is_paused": false 2024-09-20T22:29:17.284 INFO:teuthology.orchestra.run.smithi037.stdout:} 2024-09-20T22:29:17.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:17 smithi194 ceph-mon[51725]: from='client.44353 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:29:17.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:17 smithi194 ceph-mon[51725]: pgmap v287: 97 pgs: 6 active+undersized, 9 peering, 22 active+undersized+degraded, 60 active+clean; 339 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 2.4 MiB/s wr, 269 op/s; 175/1533 objects degraded (11.416%) 2024-09-20T22:29:17.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:17 smithi194 ceph-mon[51725]: from='client.44357 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:29:17.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:17 smithi194 ceph-mon[51725]: from='client.? 172.21.15.37:0/2245079432' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:29:17.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:17 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:29:17.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:17 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:29:17.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:17 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:29:17.882 INFO:teuthology.orchestra.run.smithi037.stdout: 5 263972 9.73 MB/sec execute 575 sec latency 56.718 ms 2024-09-20T22:29:17.951 INFO:teuthology.orchestra.run.smithi037.stdout:HEALTH_WARN 1 osds down; all OSDs are running squid or later but require_osd_release < squid; Degraded data redundancy: 175/1533 objects degraded (11.416%), 22 pgs degraded 2024-09-20T22:29:17.951 INFO:teuthology.orchestra.run.smithi037.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-20T22:29:17.951 INFO:teuthology.orchestra.run.smithi037.stdout: osd.7 (root=default,host=smithi194) is down 2024-09-20T22:29:17.951 INFO:teuthology.orchestra.run.smithi037.stdout:[WRN] OSD_UPGRADE_FINISHED: all OSDs are running squid or later but require_osd_release < squid 2024-09-20T22:29:17.951 INFO:teuthology.orchestra.run.smithi037.stdout: all OSDs are running squid or later but require_osd_release < squid 2024-09-20T22:29:17.951 INFO:teuthology.orchestra.run.smithi037.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 175/1533 objects degraded (11.416%), 22 pgs degraded 2024-09-20T22:29:17.951 INFO:teuthology.orchestra.run.smithi037.stdout: pg 1.0 is active+undersized+degraded, acting [0,6] 2024-09-20T22:29:17.951 INFO:teuthology.orchestra.run.smithi037.stdout: pg 2.3 is active+undersized+degraded, acting [5,2] 2024-09-20T22:29:17.951 INFO:teuthology.orchestra.run.smithi037.stdout: pg 2.5 is active+undersized+degraded, acting [0,4] 2024-09-20T22:29:17.951 INFO:teuthology.orchestra.run.smithi037.stdout: pg 2.e is active+undersized+degraded, acting [2,3] 2024-09-20T22:29:17.952 INFO:teuthology.orchestra.run.smithi037.stdout: pg 2.f is active+undersized+degraded, acting [4,0] 2024-09-20T22:29:17.952 INFO:teuthology.orchestra.run.smithi037.stdout: pg 2.12 is active+undersized+degraded, acting [5,3] 2024-09-20T22:29:17.952 INFO:teuthology.orchestra.run.smithi037.stdout: pg 2.18 is active+undersized+degraded, acting [5,4] 2024-09-20T22:29:17.952 INFO:teuthology.orchestra.run.smithi037.stdout: pg 2.1a is active+undersized+degraded, acting [6,4] 2024-09-20T22:29:17.952 INFO:teuthology.orchestra.run.smithi037.stdout: pg 2.1d is active+undersized+degraded, acting [6,0] 2024-09-20T22:29:17.952 INFO:teuthology.orchestra.run.smithi037.stdout: pg 3.9 is active+undersized+degraded, acting [4,2] 2024-09-20T22:29:17.952 INFO:teuthology.orchestra.run.smithi037.stdout: pg 3.d is active+undersized+degraded, acting [5,6] 2024-09-20T22:29:17.952 INFO:teuthology.orchestra.run.smithi037.stdout: pg 3.e is active+undersized+degraded, acting [4,1] 2024-09-20T22:29:17.952 INFO:teuthology.orchestra.run.smithi037.stdout: pg 3.f is active+undersized+degraded, acting [4,0] 2024-09-20T22:29:17.952 INFO:teuthology.orchestra.run.smithi037.stdout: pg 3.11 is active+undersized+degraded, acting [4,6] 2024-09-20T22:29:17.952 INFO:teuthology.orchestra.run.smithi037.stdout: pg 3.12 is active+undersized+degraded, acting [0,3] 2024-09-20T22:29:17.952 INFO:teuthology.orchestra.run.smithi037.stdout: pg 3.13 is active+undersized+degraded, acting [4,2] 2024-09-20T22:29:17.952 INFO:teuthology.orchestra.run.smithi037.stdout: pg 3.14 is active+undersized+degraded, acting [4,6] 2024-09-20T22:29:17.952 INFO:teuthology.orchestra.run.smithi037.stdout: pg 3.16 is active+undersized+degraded, acting [5,1] 2024-09-20T22:29:17.952 INFO:teuthology.orchestra.run.smithi037.stdout: pg 3.1b is active+undersized+degraded, acting [0,4] 2024-09-20T22:29:17.953 INFO:teuthology.orchestra.run.smithi037.stdout: pg 4.0 is active+undersized+degraded, acting [4,0] 2024-09-20T22:29:17.953 INFO:teuthology.orchestra.run.smithi037.stdout: pg 4.5 is active+undersized+degraded, acting [6,4] 2024-09-20T22:29:17.953 INFO:teuthology.orchestra.run.smithi037.stdout: pg 4.a is active+undersized+degraded, acting [6,1] 2024-09-20T22:29:18.126 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:18 smithi037 ceph-mon[65640]: from='client.? 172.21.15.37:0/2799635073' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T22:29:18.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:18 smithi194 ceph-mon[51725]: from='client.? 172.21.15.37:0/2799635073' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T22:29:18.845 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:29:18 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:29:18.650+0000 7f2637399640 -1 Falling back to public interface 2024-09-20T22:29:18.882 INFO:teuthology.orchestra.run.smithi037.stdout: 5 264284 9.72 MB/sec execute 576 sec latency 48.913 ms 2024-09-20T22:29:19.134 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:19 smithi037 ceph-mon[65640]: from='client.34446 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:29:19.134 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:19 smithi037 ceph-mon[65640]: pgmap v288: 97 pgs: 8 active+undersized, 29 active+undersized+degraded, 60 active+clean; 340 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.6 MiB/s wr, 482 op/s; 250/1632 objects degraded (15.319%) 2024-09-20T22:29:19.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:19 smithi194 ceph-mon[51725]: from='client.34446 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:29:19.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:19 smithi194 ceph-mon[51725]: pgmap v288: 97 pgs: 8 active+undersized, 29 active+undersized+degraded, 60 active+clean; 340 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.6 MiB/s wr, 482 op/s; 250/1632 objects degraded (15.319%) 2024-09-20T22:29:19.882 INFO:teuthology.orchestra.run.smithi037.stdout: 5 264627 9.71 MB/sec execute 577 sec latency 42.314 ms 2024-09-20T22:29:20.146 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:20 smithi037 ceph-mon[65640]: Health check update: Degraded data redundancy: 250/1632 objects degraded (15.319%), 29 pgs degraded (PG_DEGRADED) 2024-09-20T22:29:20.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:20 smithi194 ceph-mon[51725]: Health check update: Degraded data redundancy: 250/1632 objects degraded (15.319%), 29 pgs degraded (PG_DEGRADED) 2024-09-20T22:29:20.882 INFO:teuthology.orchestra.run.smithi037.stdout: 5 264962 9.71 MB/sec execute 578 sec latency 41.917 ms 2024-09-20T22:29:21.170 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:21 smithi037 ceph-mon[65640]: pgmap v289: 97 pgs: 8 active+undersized, 29 active+undersized+degraded, 60 active+clean; 341 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.5 MiB/s wr, 433 op/s; 252/1650 objects degraded (15.273%) 2024-09-20T22:29:21.171 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:21 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:29:21.171 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:21 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:29:21.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:21 smithi194 ceph-mon[51725]: pgmap v289: 97 pgs: 8 active+undersized, 29 active+undersized+degraded, 60 active+clean; 341 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.5 MiB/s wr, 433 op/s; 252/1650 objects degraded (15.273%) 2024-09-20T22:29:21.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:21 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:29:21.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:21 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:29:21.882 INFO:teuthology.orchestra.run.smithi037.stdout: 5 265267 9.71 MB/sec execute 579 sec latency 49.860 ms 2024-09-20T22:29:22.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:22 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:29:22.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:22 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:29:22.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:22 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:29:22.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:22 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:29:22.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:22 smithi194 ceph-mon[51725]: pgmap v290: 97 pgs: 8 active+undersized, 29 active+undersized+degraded, 60 active+clean; 343 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 4.0 MiB/s wr, 353 op/s; 254/1656 objects degraded (15.338%) 2024-09-20T22:29:22.696 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:22 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:29:22.696 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:22 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:29:22.696 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:22 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:29:22.696 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:22 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:29:22.696 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:22 smithi037 ceph-mon[65640]: pgmap v290: 97 pgs: 8 active+undersized, 29 active+undersized+degraded, 60 active+clean; 343 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 4.0 MiB/s wr, 353 op/s; 254/1656 objects degraded (15.338%) 2024-09-20T22:29:22.882 INFO:teuthology.orchestra.run.smithi037.stdout: 5 265707 9.71 MB/sec execute 580 sec latency 40.146 ms 2024-09-20T22:29:23.882 INFO:teuthology.orchestra.run.smithi037.stdout: 5 266041 9.70 MB/sec execute 581 sec latency 36.530 ms 2024-09-20T22:29:24.882 INFO:teuthology.orchestra.run.smithi037.stdout: 5 266380 9.71 MB/sec execute 582 sec latency 36.527 ms 2024-09-20T22:29:25.009 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:24 smithi037 ceph-mon[65640]: pgmap v291: 97 pgs: 8 active+undersized, 29 active+undersized+degraded, 60 active+clean; 344 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.3 MiB/s wr, 416 op/s; 229/1581 objects degraded (14.485%) 2024-09-20T22:29:25.010 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:24 smithi037 ceph-mon[65640]: Health check update: Degraded data redundancy: 229/1581 objects degraded (14.485%), 29 pgs degraded (PG_DEGRADED) 2024-09-20T22:29:25.171 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:24 smithi194 ceph-mon[51725]: pgmap v291: 97 pgs: 8 active+undersized, 29 active+undersized+degraded, 60 active+clean; 344 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.3 MiB/s wr, 416 op/s; 229/1581 objects degraded (14.485%) 2024-09-20T22:29:25.171 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:24 smithi194 ceph-mon[51725]: Health check update: Degraded data redundancy: 229/1581 objects degraded (14.485%), 29 pgs degraded (PG_DEGRADED) 2024-09-20T22:29:25.882 INFO:teuthology.orchestra.run.smithi037.stdout: 5 266781 9.72 MB/sec execute 583 sec latency 46.076 ms 2024-09-20T22:29:26.269 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:26 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:29:26.269 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:26 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:29:26.269 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:26 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:29:26.270 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:26 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:29:26.270 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:26 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:29:26.270 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:26 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:29:26.270 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:26 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:29:26.270 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:26 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:29:26.270 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:26 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:29:26.270 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:26 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:29:26.270 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:26 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:29:26.270 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:26 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:29:26.270 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:26 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-09-20T22:29:26.270 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:26 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-09-20T22:29:26.270 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:26 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-09-20T22:29:26.270 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:26 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-09-20T22:29:26.271 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:26 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-09-20T22:29:26.271 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:26 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-09-20T22:29:26.271 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:26 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-09-20T22:29:26.271 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:26 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-09-20T22:29:26.271 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:26 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-09-20T22:29:26.271 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:26 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-09-20T22:29:26.271 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:26 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-09-20T22:29:26.271 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:26 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-09-20T22:29:26.271 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:26 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-09-20T22:29:26.271 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:26 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-09-20T22:29:26.271 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:26 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-09-20T22:29:26.271 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:26 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-09-20T22:29:26.271 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:26 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-09-20T22:29:26.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:26 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:29:26.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:26 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:29:26.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:26 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:29:26.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:26 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:29:26.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:26 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:29:26.596 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:26 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:29:26.596 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:26 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:29:26.596 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:26 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:29:26.596 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:26 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:29:26.596 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:26 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:29:26.596 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:26 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:29:26.596 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:26 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:29:26.596 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:26 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-09-20T22:29:26.596 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:26 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-09-20T22:29:26.596 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:26 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-09-20T22:29:26.596 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:26 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-09-20T22:29:26.596 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:26 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-09-20T22:29:26.596 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:26 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-09-20T22:29:26.596 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:26 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-09-20T22:29:26.596 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:26 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-09-20T22:29:26.596 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:26 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-09-20T22:29:26.597 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:26 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-09-20T22:29:26.597 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:26 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-09-20T22:29:26.597 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:26 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-09-20T22:29:26.597 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:26 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-09-20T22:29:26.597 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:26 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-09-20T22:29:26.597 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:26 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-09-20T22:29:26.597 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:26 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-09-20T22:29:26.597 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:26 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-09-20T22:29:26.882 INFO:teuthology.orchestra.run.smithi037.stdout: 5 267214 9.73 MB/sec execute 584 sec latency 36.901 ms 2024-09-20T22:29:27.279 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:27 smithi037 ceph-mon[65640]: Upgrade: Setting container_image for all osd 2024-09-20T22:29:27.279 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:27 smithi037 ceph-mon[65640]: Upgrade: Setting require_osd_release to 19 squid 2024-09-20T22:29:27.279 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:27 smithi037 ceph-mon[65640]: pgmap v292: 97 pgs: 8 active+undersized, 29 active+undersized+degraded, 60 active+clean; 343 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 5.0 MiB/s wr, 500 op/s; 224/1590 objects degraded (14.088%) 2024-09-20T22:29:27.279 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:27 smithi037 ceph-mon[65640]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-09-20T22:29:27.279 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:27 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-09-20T22:29:27.279 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:27 smithi037 ceph-mon[65640]: osdmap e151: 8 total, 7 up, 8 in 2024-09-20T22:29:27.279 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:27 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi037.sdtedi"]}]: dispatch 2024-09-20T22:29:27.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:27 smithi194 ceph-mon[51725]: Upgrade: Setting container_image for all osd 2024-09-20T22:29:27.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:27 smithi194 ceph-mon[51725]: Upgrade: Setting require_osd_release to 19 squid 2024-09-20T22:29:27.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:27 smithi194 ceph-mon[51725]: pgmap v292: 97 pgs: 8 active+undersized, 29 active+undersized+degraded, 60 active+clean; 343 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 5.0 MiB/s wr, 500 op/s; 224/1590 objects degraded (14.088%) 2024-09-20T22:29:27.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:27 smithi194 ceph-mon[51725]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-09-20T22:29:27.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:27 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-09-20T22:29:27.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:27 smithi194 ceph-mon[51725]: osdmap e151: 8 total, 7 up, 8 in 2024-09-20T22:29:27.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:27 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi037.sdtedi"]}]: dispatch 2024-09-20T22:29:27.882 INFO:teuthology.orchestra.run.smithi037.stdout: 5 267493 9.72 MB/sec execute 585 sec latency 54.615 ms 2024-09-20T22:29:28.302 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:28 smithi037 ceph-mon[65640]: Upgrade: It appears safe to stop mds.foofs.smithi037.sdtedi 2024-09-20T22:29:28.496 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:28 smithi194 ceph-mon[51725]: Upgrade: It appears safe to stop mds.foofs.smithi037.sdtedi 2024-09-20T22:29:28.882 INFO:teuthology.orchestra.run.smithi037.stdout: 5 267703 9.71 MB/sec execute 586 sec latency 56.236 ms 2024-09-20T22:29:29.094 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:29:28 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:29:28.752+0000 7f2637399640 -1 osd.7 0 read_superblock omap replica is missing. 2024-09-20T22:29:29.331 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:29 smithi037 ceph-mon[65640]: pgmap v294: 97 pgs: 8 active+undersized, 29 active+undersized+degraded, 60 active+clean; 344 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.0 MiB/s wr, 502 op/s; 231/1644 objects degraded (14.051%) 2024-09-20T22:29:29.507 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:29 smithi194 ceph-mon[51725]: pgmap v294: 97 pgs: 8 active+undersized, 29 active+undersized+degraded, 60 active+clean; 344 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.0 MiB/s wr, 502 op/s; 231/1644 objects degraded (14.051%) 2024-09-20T22:29:29.882 INFO:teuthology.orchestra.run.smithi037.stdout: 5 267974 9.70 MB/sec execute 587 sec latency 112.532 ms 2024-09-20T22:29:30.387 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:30 smithi037 ceph-mon[65640]: Health check update: Degraded data redundancy: 231/1644 objects degraded (14.051%), 29 pgs degraded (PG_DEGRADED) 2024-09-20T22:29:30.387 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:30 smithi037 ceph-mon[65640]: Upgrade: Updating mds.foofs.smithi037.sdtedi 2024-09-20T22:29:30.387 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:30 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:29:30.388 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:30 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi037.sdtedi", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-20T22:29:30.388 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:30 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:29:30.388 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:30 smithi037 ceph-mon[65640]: Deploying daemon mds.foofs.smithi037.sdtedi on smithi037 2024-09-20T22:29:30.388 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:30 smithi037 ceph-mon[65640]: pgmap v295: 97 pgs: 8 active+undersized, 29 active+undersized+degraded, 60 active+clean; 344 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.0 MiB/s wr, 504 op/s; 231/1671 objects degraded (13.824%) 2024-09-20T22:29:30.388 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:30 smithi037 ceph-mon[65640]: from='osd.7 [v2:172.21.15.194:6824/197673991,v1:172.21.15.194:6825/197673991]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-20T22:29:30.388 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:30 smithi037 ceph-mon[65640]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-20T22:29:30.594 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:29:30 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:29:30.191+0000 7f2637399640 -1 osd.7 148 log_to_monitors true 2024-09-20T22:29:30.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:30 smithi194 ceph-mon[51725]: Health check update: Degraded data redundancy: 231/1644 objects degraded (14.051%), 29 pgs degraded (PG_DEGRADED) 2024-09-20T22:29:30.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:30 smithi194 ceph-mon[51725]: Upgrade: Updating mds.foofs.smithi037.sdtedi 2024-09-20T22:29:30.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:30 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:29:30.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:30 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi037.sdtedi", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-20T22:29:30.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:30 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:29:30.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:30 smithi194 ceph-mon[51725]: Deploying daemon mds.foofs.smithi037.sdtedi on smithi037 2024-09-20T22:29:30.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:30 smithi194 ceph-mon[51725]: pgmap v295: 97 pgs: 8 active+undersized, 29 active+undersized+degraded, 60 active+clean; 344 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.0 MiB/s wr, 504 op/s; 231/1671 objects degraded (13.824%) 2024-09-20T22:29:30.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:30 smithi194 ceph-mon[51725]: from='osd.7 [v2:172.21.15.194:6824/197673991,v1:172.21.15.194:6825/197673991]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-20T22:29:30.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:30 smithi194 ceph-mon[51725]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-20T22:29:30.882 INFO:teuthology.orchestra.run.smithi037.stdout: 5 268195 9.70 MB/sec execute 588 sec latency 127.126 ms 2024-09-20T22:29:31.344 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:29:31 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:29:31.184+0000 7f262b13c640 -1 osd.7 148 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-20T22:29:31.696 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:31 smithi037 ceph-mon[65640]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-20T22:29:31.696 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:31 smithi037 ceph-mon[65640]: from='osd.7 [v2:172.21.15.194:6824/197673991,v1:172.21.15.194:6825/197673991]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2024-09-20T22:29:31.696 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:31 smithi037 ceph-mon[65640]: osdmap e152: 8 total, 7 up, 8 in 2024-09-20T22:29:31.696 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:31 smithi037 ceph-mon[65640]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2024-09-20T22:29:31.696 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:31 smithi037 ceph-mon[65640]: from='osd.7 ' entity='osd.7' 2024-09-20T22:29:31.882 INFO:teuthology.orchestra.run.smithi037.stdout: 5 268395 9.69 MB/sec execute 589 sec latency 219.558 ms 2024-09-20T22:29:32.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:31 smithi194 ceph-mon[51725]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-20T22:29:32.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:31 smithi194 ceph-mon[51725]: from='osd.7 [v2:172.21.15.194:6824/197673991,v1:172.21.15.194:6825/197673991]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2024-09-20T22:29:32.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:31 smithi194 ceph-mon[51725]: osdmap e152: 8 total, 7 up, 8 in 2024-09-20T22:29:32.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:31 smithi194 ceph-mon[51725]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2024-09-20T22:29:32.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:31 smithi194 ceph-mon[51725]: from='osd.7 ' entity='osd.7' 2024-09-20T22:29:32.882 INFO:teuthology.orchestra.run.smithi037.stdout: 5 268521 9.68 MB/sec execute 590 sec latency 689.456 ms 2024-09-20T22:29:32.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:32 smithi037 ceph-mon[65640]: pgmap v297: 97 pgs: 8 active+undersized, 29 active+undersized+degraded, 60 active+clean; 342 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 3.7 KiB/s rd, 4.3 MiB/s wr, 373 op/s; 232/1671 objects degraded (13.884%) 2024-09-20T22:29:32.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:32 smithi037 ceph-mon[65640]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-20T22:29:32.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:32 smithi037 ceph-mon[65640]: osd.7 [v2:172.21.15.194:6824/197673991,v1:172.21.15.194:6825/197673991] boot 2024-09-20T22:29:32.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:32 smithi037 ceph-mon[65640]: osdmap e153: 8 total, 8 up, 8 in 2024-09-20T22:29:32.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:32 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-20T22:29:33.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:32 smithi194 ceph-mon[51725]: pgmap v297: 97 pgs: 8 active+undersized, 29 active+undersized+degraded, 60 active+clean; 342 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 3.7 KiB/s rd, 4.3 MiB/s wr, 373 op/s; 232/1671 objects degraded (13.884%) 2024-09-20T22:29:33.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:32 smithi194 ceph-mon[51725]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-20T22:29:33.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:32 smithi194 ceph-mon[51725]: osd.7 [v2:172.21.15.194:6824/197673991,v1:172.21.15.194:6825/197673991] boot 2024-09-20T22:29:33.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:32 smithi194 ceph-mon[51725]: osdmap e153: 8 total, 8 up, 8 in 2024-09-20T22:29:33.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:32 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-20T22:29:33.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:33 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-mon-smithi037[65636]: 2024-09-20T22:29:33.099+0000 7f97ffa2b640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-20T22:29:33.594 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:29:33 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-5[58907]: 2024-09-20T22:29:33.229+0000 7f63d57c4640 -1 bdev(0x5599db6a6c00 /var/lib/ceph/osd/ceph-5/block) aio_submit retries 3 2024-09-20T22:29:33.595 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:29:33 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-5[58907]: 2024-09-20T22:29:33.233+0000 7f63d3fc1640 -1 bdev(0x5599db6a6c00 /var/lib/ceph/osd/ceph-5/block) aio_submit retries 13 2024-09-20T22:29:33.882 INFO:teuthology.orchestra.run.smithi037.stdout: 5 268521 9.66 MB/sec execute 591 sec latency 1689.504 ms 2024-09-20T22:29:34.091 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:33 smithi037 ceph-mon[65640]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-20T22:29:34.092 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:33 smithi037 ceph-mon[65640]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-20T22:29:34.092 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:33 smithi037 ceph-mon[65640]: osdmap e154: 8 total, 8 up, 8 in 2024-09-20T22:29:34.092 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:33 smithi037 ceph-mon[65640]: Standby daemon mds.foofs.smithi194.lkroso assigned to filesystem foofs as rank 0 2024-09-20T22:29:34.092 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:33 smithi037 ceph-mon[65640]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-09-20T22:29:34.092 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:33 smithi037 ceph-mon[65640]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-20T22:29:34.092 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:33 smithi037 ceph-mon[65640]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-09-20T22:29:34.092 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:33 smithi037 ceph-mon[65640]: fsmap foofs:1/1 {0=foofs.smithi194.lkroso=up:replay} 2024-09-20T22:29:34.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:33 smithi194 ceph-mon[51725]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-20T22:29:34.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:33 smithi194 ceph-mon[51725]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-20T22:29:34.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:33 smithi194 ceph-mon[51725]: osdmap e154: 8 total, 8 up, 8 in 2024-09-20T22:29:34.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:33 smithi194 ceph-mon[51725]: Standby daemon mds.foofs.smithi194.lkroso assigned to filesystem foofs as rank 0 2024-09-20T22:29:34.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:33 smithi194 ceph-mon[51725]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-09-20T22:29:34.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:33 smithi194 ceph-mon[51725]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-20T22:29:34.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:33 smithi194 ceph-mon[51725]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-09-20T22:29:34.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:33 smithi194 ceph-mon[51725]: fsmap foofs:1/1 {0=foofs.smithi194.lkroso=up:replay} 2024-09-20T22:29:34.882 INFO:teuthology.orchestra.run.smithi037.stdout: 5 268523 9.65 MB/sec execute 592 sec latency 2011.673 ms 2024-09-20T22:29:35.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:35 smithi037 ceph-mon[65640]: pgmap v300: 97 pgs: 8 active+undersized, 29 active+undersized+degraded, 60 active+clean; 343 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.3 KiB/s rd, 3.6 MiB/s wr, 373 op/s; 217/1539 objects degraded (14.100%) 2024-09-20T22:29:35.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:35 smithi037 ceph-mon[65640]: osdmap e155: 8 total, 8 up, 8 in 2024-09-20T22:29:35.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:35 smithi037 ceph-mon[65640]: Health check update: Degraded data redundancy: 232/1671 objects degraded (13.884%), 29 pgs degraded (PG_DEGRADED) 2024-09-20T22:29:35.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:35 smithi037 ceph-mon[65640]: osdmap e156: 8 total, 8 up, 8 in 2024-09-20T22:29:35.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:35 smithi194 ceph-mon[51725]: pgmap v300: 97 pgs: 8 active+undersized, 29 active+undersized+degraded, 60 active+clean; 343 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.3 KiB/s rd, 3.6 MiB/s wr, 373 op/s; 217/1539 objects degraded (14.100%) 2024-09-20T22:29:35.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:35 smithi194 ceph-mon[51725]: osdmap e155: 8 total, 8 up, 8 in 2024-09-20T22:29:35.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:35 smithi194 ceph-mon[51725]: Health check update: Degraded data redundancy: 232/1671 objects degraded (13.884%), 29 pgs degraded (PG_DEGRADED) 2024-09-20T22:29:35.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:35 smithi194 ceph-mon[51725]: osdmap e156: 8 total, 8 up, 8 in 2024-09-20T22:29:35.882 INFO:teuthology.orchestra.run.smithi037.stdout: 5 268523 9.63 MB/sec execute 593 sec latency 1694.528 ms 2024-09-20T22:29:36.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:36 smithi037 ceph-mon[65640]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-09-20T22:29:36.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:36 smithi037 ceph-mon[65640]: mds.? [v2:172.21.15.37:6834/1645917022,v1:172.21.15.37:6835/1645917022] up:boot 2024-09-20T22:29:36.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:36 smithi037 ceph-mon[65640]: fsmap foofs:1/1 {0=foofs.smithi194.lkroso=up:replay} 1 up:standby 2024-09-20T22:29:36.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:36 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi037.sdtedi"}]: dispatch 2024-09-20T22:29:36.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:36 smithi037 ceph-mon[65640]: osdmap e157: 8 total, 8 up, 8 in 2024-09-20T22:29:36.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:36 smithi194 ceph-mon[51725]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-09-20T22:29:36.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:36 smithi194 ceph-mon[51725]: mds.? [v2:172.21.15.37:6834/1645917022,v1:172.21.15.37:6835/1645917022] up:boot 2024-09-20T22:29:36.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:36 smithi194 ceph-mon[51725]: fsmap foofs:1/1 {0=foofs.smithi194.lkroso=up:replay} 1 up:standby 2024-09-20T22:29:36.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:36 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi037.sdtedi"}]: dispatch 2024-09-20T22:29:36.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:36 smithi194 ceph-mon[51725]: osdmap e157: 8 total, 8 up, 8 in 2024-09-20T22:29:36.882 INFO:teuthology.orchestra.run.smithi037.stdout: 5 268523 9.61 MB/sec execute 594 sec latency 2694.558 ms 2024-09-20T22:29:37.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:37 smithi037 ceph-mon[65640]: pgmap v304: 97 pgs: 5 peering, 1 active+undersized+remapped, 2 active+undersized, 12 active+undersized+degraded, 77 active+clean; 336 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 1.8 MiB/s wr, 95 op/s; 64/1449 objects degraded (4.417%); 1.4 MiB/s, 29 objects/s recovering 2024-09-20T22:29:37.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:37 smithi037 ceph-mon[65640]: osdmap e158: 8 total, 8 up, 8 in 2024-09-20T22:29:37.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:37 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:29:37.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:37 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:29:37.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:37 smithi194 ceph-mon[51725]: pgmap v304: 97 pgs: 5 peering, 1 active+undersized+remapped, 2 active+undersized, 12 active+undersized+degraded, 77 active+clean; 336 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 1.8 MiB/s wr, 95 op/s; 64/1449 objects degraded (4.417%); 1.4 MiB/s, 29 objects/s recovering 2024-09-20T22:29:37.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:37 smithi194 ceph-mon[51725]: osdmap e158: 8 total, 8 up, 8 in 2024-09-20T22:29:37.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:37 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:29:37.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:37 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:29:37.882 INFO:teuthology.orchestra.run.smithi037.stdout: 5 268523 9.60 MB/sec execute 595 sec latency 3694.585 ms 2024-09-20T22:29:38.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:38 smithi037 ceph-mon[65640]: reconnect by client.24363 172.21.15.194:0/2491128762 after 0 2024-09-20T22:29:38.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:38 smithi037 ceph-mon[65640]: reconnect by client.14550 172.21.15.37:0/859528995 after 0.00099998 2024-09-20T22:29:38.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:38 smithi037 ceph-mon[65640]: mds.? [v2:172.21.15.194:6832/1702667629,v1:172.21.15.194:6833/1702667629] up:reconnect 2024-09-20T22:29:38.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:38 smithi037 ceph-mon[65640]: fsmap foofs:1/1 {0=foofs.smithi194.lkroso=up:reconnect} 1 up:standby 2024-09-20T22:29:38.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:38 smithi194 ceph-mon[51725]: reconnect by client.24363 172.21.15.194:0/2491128762 after 0 2024-09-20T22:29:38.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:38 smithi194 ceph-mon[51725]: reconnect by client.14550 172.21.15.37:0/859528995 after 0.00099998 2024-09-20T22:29:38.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:38 smithi194 ceph-mon[51725]: mds.? [v2:172.21.15.194:6832/1702667629,v1:172.21.15.194:6833/1702667629] up:reconnect 2024-09-20T22:29:38.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:38 smithi194 ceph-mon[51725]: fsmap foofs:1/1 {0=foofs.smithi194.lkroso=up:reconnect} 1 up:standby 2024-09-20T22:29:38.882 INFO:teuthology.orchestra.run.smithi037.stdout: 5 268523 9.58 MB/sec execute 596 sec latency 4694.642 ms 2024-09-20T22:29:39.377 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:39 smithi037 ceph-mon[65640]: pgmap v306: 97 pgs: 5 peering, 1 active+undersized+remapped, 91 active+clean; 336 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 47 MiB/s rd, 817 KiB/s wr, 84 op/s; 2.4 MiB/s, 48 objects/s recovering 2024-09-20T22:29:39.377 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:39 smithi037 ceph-mon[65640]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 64/1449 objects degraded (4.417%), 12 pgs degraded) 2024-09-20T22:29:39.377 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:39 smithi037 ceph-mon[65640]: mds.? [v2:172.21.15.194:6832/1702667629,v1:172.21.15.194:6833/1702667629] up:rejoin 2024-09-20T22:29:39.377 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:39 smithi037 ceph-mon[65640]: fsmap foofs:1/1 {0=foofs.smithi194.lkroso=up:rejoin} 1 up:standby 2024-09-20T22:29:39.377 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:39 smithi037 ceph-mon[65640]: daemon mds.foofs.smithi194.lkroso is now active in filesystem foofs as rank 0 2024-09-20T22:29:39.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:39 smithi194 ceph-mon[51725]: pgmap v306: 97 pgs: 5 peering, 1 active+undersized+remapped, 91 active+clean; 336 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 47 MiB/s rd, 817 KiB/s wr, 84 op/s; 2.4 MiB/s, 48 objects/s recovering 2024-09-20T22:29:39.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:39 smithi194 ceph-mon[51725]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 64/1449 objects degraded (4.417%), 12 pgs degraded) 2024-09-20T22:29:39.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:39 smithi194 ceph-mon[51725]: mds.? [v2:172.21.15.194:6832/1702667629,v1:172.21.15.194:6833/1702667629] up:rejoin 2024-09-20T22:29:39.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:39 smithi194 ceph-mon[51725]: fsmap foofs:1/1 {0=foofs.smithi194.lkroso=up:rejoin} 1 up:standby 2024-09-20T22:29:39.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:39 smithi194 ceph-mon[51725]: daemon mds.foofs.smithi194.lkroso is now active in filesystem foofs as rank 0 2024-09-20T22:29:39.882 INFO:teuthology.orchestra.run.smithi037.stdout: 5 268703 9.58 MB/sec execute 597 sec latency 4957.878 ms 2024-09-20T22:29:40.165 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:40 smithi037 ceph-mon[65640]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-20T22:29:40.166 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:40 smithi037 ceph-mon[65640]: Cluster is now healthy 2024-09-20T22:29:40.166 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:40 smithi037 ceph-mon[65640]: mds.? [v2:172.21.15.194:6832/1702667629,v1:172.21.15.194:6833/1702667629] up:active 2024-09-20T22:29:40.166 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:40 smithi037 ceph-mon[65640]: fsmap foofs:1 {0=foofs.smithi194.lkroso=up:active} 1 up:standby 2024-09-20T22:29:40.416 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:40 smithi194 ceph-mon[51725]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-20T22:29:40.416 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:40 smithi194 ceph-mon[51725]: Cluster is now healthy 2024-09-20T22:29:40.417 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:40 smithi194 ceph-mon[51725]: mds.? [v2:172.21.15.194:6832/1702667629,v1:172.21.15.194:6833/1702667629] up:active 2024-09-20T22:29:40.417 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:40 smithi194 ceph-mon[51725]: fsmap foofs:1 {0=foofs.smithi194.lkroso=up:active} 1 up:standby 2024-09-20T22:29:40.882 INFO:teuthology.orchestra.run.smithi037.stdout: 5 268863 9.56 MB/sec execute 598 sec latency 117.138 ms 2024-09-20T22:29:41.206 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:41 smithi037 ceph-mon[65640]: pgmap v307: 97 pgs: 4 peering, 1 active+undersized+remapped, 92 active+clean; 336 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 33 MiB/s rd, 567 KiB/s wr, 61 op/s; 1.7 MiB/s, 33 objects/s recovering 2024-09-20T22:29:41.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:41 smithi194 ceph-mon[51725]: pgmap v307: 97 pgs: 4 peering, 1 active+undersized+remapped, 92 active+clean; 336 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 33 MiB/s rd, 567 KiB/s wr, 61 op/s; 1.7 MiB/s, 33 objects/s recovering 2024-09-20T22:29:41.882 INFO:teuthology.orchestra.run.smithi037.stdout: 5 269087 9.55 MB/sec execute 599 sec latency 150.516 ms 2024-09-20T22:29:42.882 INFO:teuthology.orchestra.run.smithi037.stdout: 5 cleanup 600 sec 2024-09-20T22:29:43.085 INFO:teuthology.orchestra.run.smithi037.stdout: 0 cleanup 600 sec 2024-09-20T22:29:43.085 INFO:teuthology.orchestra.run.smithi037.stdout: 2024-09-20T22:29:43.085 INFO:teuthology.orchestra.run.smithi037.stdout: Operation Count AvgLat MaxLat 2024-09-20T22:29:43.085 INFO:teuthology.orchestra.run.smithi037.stdout: ---------------------------------------- 2024-09-20T22:29:43.085 INFO:teuthology.orchestra.run.smithi037.stdout: NTCreateX 184436 4.650 4957.874 2024-09-20T22:29:43.085 INFO:teuthology.orchestra.run.smithi037.stdout: Close 135353 0.149 10.656 2024-09-20T22:29:43.085 INFO:teuthology.orchestra.run.smithi037.stdout: Rename 7795 9.542 219.551 2024-09-20T22:29:43.085 INFO:teuthology.orchestra.run.smithi037.stdout: Unlink 37318 6.690 2041.890 2024-09-20T22:29:43.085 INFO:teuthology.orchestra.run.smithi037.stdout: Qpathinfo 167175 1.259 198.221 2024-09-20T22:29:43.085 INFO:teuthology.orchestra.run.smithi037.stdout: Qfileinfo 29080 0.007 0.808 2024-09-20T22:29:43.085 INFO:teuthology.orchestra.run.smithi037.stdout: Qfsinfo 30608 0.646 18.605 2024-09-20T22:29:43.085 INFO:teuthology.orchestra.run.smithi037.stdout: Sfileinfo 15004 12.716 266.216 2024-09-20T22:29:43.086 INFO:teuthology.orchestra.run.smithi037.stdout: Find 64510 1.072 38.180 2024-09-20T22:29:43.086 INFO:teuthology.orchestra.run.smithi037.stdout: WriteX 90938 14.307 4954.719 2024-09-20T22:29:43.086 INFO:teuthology.orchestra.run.smithi037.stdout: ReadX 288351 0.011 2.774 2024-09-20T22:29:43.086 INFO:teuthology.orchestra.run.smithi037.stdout: LockX 596 0.411 3.760 2024-09-20T22:29:43.086 INFO:teuthology.orchestra.run.smithi037.stdout: UnlockX 596 0.353 3.367 2024-09-20T22:29:43.086 INFO:teuthology.orchestra.run.smithi037.stdout: Flush 12911 0.003 0.059 2024-09-20T22:29:43.086 INFO:teuthology.orchestra.run.smithi037.stdout: 2024-09-20T22:29:43.086 INFO:teuthology.orchestra.run.smithi037.stdout:Throughput 9.55482 MB/sec 5 clients 5 procs max_latency=4957.878 ms 2024-09-20T22:29:43.088 DEBUG:teuthology.orchestra.run.smithi037:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'umount /mnt/foo' 2024-09-20T22:29:43.130 DEBUG:teuthology.orchestra.run.smithi037:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'while ! mount -t nfs $(hostname):/fake /mnt/foo ; do sleep 5 ; done' 2024-09-20T22:29:43.249 DEBUG:teuthology.orchestra.run.smithi037:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'cd /mnt/foo && dbench 5 -t 5' 2024-09-20T22:29:43.275 INFO:teuthology.orchestra.run.smithi037.stdout:dbench version 4.00 - Copyright Andrew Tridgell 1999-2004 2024-09-20T22:29:43.275 INFO:teuthology.orchestra.run.smithi037.stdout: 2024-09-20T22:29:43.275 INFO:teuthology.orchestra.run.smithi037.stdout:Running for 5 seconds with load '/usr/share/dbench/client.txt' and minimum warmup 1 secs 2024-09-20T22:29:43.276 INFO:teuthology.orchestra.run.smithi037.stdout:3 of 5 processes prepared for launch 0 sec 2024-09-20T22:29:43.376 INFO:teuthology.orchestra.run.smithi037.stdout:5 of 5 processes prepared for launch 0 sec 2024-09-20T22:29:43.376 INFO:teuthology.orchestra.run.smithi037.stdout:releasing clients 2024-09-20T22:29:43.377 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:43 smithi037 ceph-mon[65640]: pgmap v308: 97 pgs: 97 active+clean; 336 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 30 MiB/s rd, 532 KiB/s wr, 65 op/s; 1.5 MiB/s, 41 objects/s recovering 2024-09-20T22:29:43.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:43 smithi194 ceph-mon[51725]: pgmap v308: 97 pgs: 97 active+clean; 336 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 30 MiB/s rd, 532 KiB/s wr, 65 op/s; 1.5 MiB/s, 41 objects/s recovering 2024-09-20T22:29:45.295 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:45 smithi037 ceph-mon[65640]: pgmap v309: 97 pgs: 97 active+clean; 338 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 604 KiB/s wr, 111 op/s; 1.1 MiB/s, 17 objects/s recovering 2024-09-20T22:29:45.376 INFO:teuthology.orchestra.run.smithi037.stdout: 5 222 11.70 MB/sec execute 1 sec latency 327.907 ms 2024-09-20T22:29:45.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:45 smithi194 ceph-mon[51725]: pgmap v309: 97 pgs: 97 active+clean; 338 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 604 KiB/s wr, 111 op/s; 1.1 MiB/s, 17 objects/s recovering 2024-09-20T22:29:46.376 INFO:teuthology.orchestra.run.smithi037.stdout: 5 297 11.96 MB/sec execute 2 sec latency 140.250 ms 2024-09-20T22:29:46.398 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:46 smithi037 ceph-mon[65640]: daemon mds.foofs.smithi037.sdtedi restarted 2024-09-20T22:29:46.398 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:46 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:29:46.398 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:46 smithi037 ceph-mon[65640]: osdmap e159: 8 total, 8 up, 8 in 2024-09-20T22:29:46.398 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:46 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:29:46.398 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:46 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:29:46.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:46 smithi194 ceph-mon[51725]: daemon mds.foofs.smithi037.sdtedi restarted 2024-09-20T22:29:46.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:46 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:29:46.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:46 smithi194 ceph-mon[51725]: osdmap e159: 8 total, 8 up, 8 in 2024-09-20T22:29:46.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:46 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:29:46.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:46 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:29:47.215 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:47 smithi037 ceph-mon[65640]: pgmap v311: 97 pgs: 97 active+clean; 336 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 2.9 MiB/s wr, 139 op/s; 658 KiB/s, 21 objects/s recovering 2024-09-20T22:29:47.215 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:47 smithi037 ceph-mon[65640]: mds.? [v2:172.21.15.37:6834/1667079263,v1:172.21.15.37:6835/1667079263] up:boot 2024-09-20T22:29:47.215 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:47 smithi037 ceph-mon[65640]: fsmap foofs:1 {0=foofs.smithi194.lkroso=up:active} 1 up:standby 2024-09-20T22:29:47.215 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:47 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi037.sdtedi"}]: dispatch 2024-09-20T22:29:47.215 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:47 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:29:47.376 INFO:teuthology.orchestra.run.smithi037.stdout: 5 377 12.05 MB/sec execute 3 sec latency 161.900 ms 2024-09-20T22:29:47.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:47 smithi194 ceph-mon[51725]: pgmap v311: 97 pgs: 97 active+clean; 336 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 2.9 MiB/s wr, 139 op/s; 658 KiB/s, 21 objects/s recovering 2024-09-20T22:29:47.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:47 smithi194 ceph-mon[51725]: mds.? [v2:172.21.15.37:6834/1667079263,v1:172.21.15.37:6835/1667079263] up:boot 2024-09-20T22:29:47.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:47 smithi194 ceph-mon[51725]: fsmap foofs:1 {0=foofs.smithi194.lkroso=up:active} 1 up:standby 2024-09-20T22:29:47.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:47 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi037.sdtedi"}]: dispatch 2024-09-20T22:29:47.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:47 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:29:48.376 INFO:teuthology.orchestra.run.smithi037.stdout: 5 457 12.29 MB/sec execute 4 sec latency 150.769 ms 2024-09-20T22:29:48.402 INFO:teuthology.orchestra.run.smithi037.stdout:true 2024-09-20T22:29:49.310 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:49 smithi037 ceph-mon[65640]: pgmap v312: 97 pgs: 97 active+clean; 321 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 6.2 MiB/s wr, 262 op/s; 175 KiB/s, 11 objects/s recovering 2024-09-20T22:29:49.355 INFO:teuthology.orchestra.run.smithi037.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T22:29:49.355 INFO:teuthology.orchestra.run.smithi037.stdout:alertmanager.smithi037 smithi037 *:9093,9094 running (9m) 2m ago 18m 25.4M - 0.25.0 c8568f914cd2 d1f99b3cd707 2024-09-20T22:29:49.356 INFO:teuthology.orchestra.run.smithi037.stdout:crash.smithi037 smithi037 running (6m) 2m ago 18m 6895k - 19.1.1-367-g9e2ac70b cc9251b744a3 77e510d8bdb1 2024-09-20T22:29:49.356 INFO:teuthology.orchestra.run.smithi037.stdout:crash.smithi194 smithi194 running (6m) 28s ago 18m 6895k - 19.1.1-367-g9e2ac70b cc9251b744a3 4d169d1309bc 2024-09-20T22:29:49.356 INFO:teuthology.orchestra.run.smithi037.stdout:grafana.smithi037 smithi037 *:3000 running (9m) 2m ago 18m 53.2M - dad864ee21e9 2306f3c2da19 2024-09-20T22:29:49.356 INFO:teuthology.orchestra.run.smithi037.stdout:mds.foofs.smithi037.sdtedi smithi037 starting - - - - 2024-09-20T22:29:49.356 INFO:teuthology.orchestra.run.smithi037.stdout:mds.foofs.smithi194.lkroso smithi194 running (12m) 28s ago 12m 16.7M - 17.2.0 e1d6a67b021e 17736de30008 2024-09-20T22:29:49.356 INFO:teuthology.orchestra.run.smithi037.stdout:mgr.smithi037.lykblh smithi037 *:8443,9283,8765 running (8m) 2m ago 19m 505M - 19.1.1-367-g9e2ac70b cc9251b744a3 181e46fd9ba6 2024-09-20T22:29:49.356 INFO:teuthology.orchestra.run.smithi037.stdout:mgr.smithi194.bkcglt smithi194 *:8443,9283,8765 running (7m) 28s ago 17m 444M - 19.1.1-367-g9e2ac70b cc9251b744a3 1fa610c97517 2024-09-20T22:29:49.356 INFO:teuthology.orchestra.run.smithi037.stdout:mon.smithi037 smithi037 running (7m) 2m ago 20m 52.0M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 c0229eceb527 2024-09-20T22:29:49.356 INFO:teuthology.orchestra.run.smithi037.stdout:mon.smithi194 smithi194 running (7m) 28s ago 17m 49.1M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 97854807ba91 2024-09-20T22:29:49.356 INFO:teuthology.orchestra.run.smithi037.stdout:nfs.foo.0.0.smithi037.spabdc smithi037 *:2049 running (12m) 2m ago 12m 280M - 4.0 e1d6a67b021e 858ff9d558a2 2024-09-20T22:29:49.356 INFO:teuthology.orchestra.run.smithi037.stdout:nfs.foo.1.0.smithi194.qzjyej smithi194 *:2049 running (12m) 28s ago 12m 90.4M - 4.0 e1d6a67b021e c07fe0eb7c75 2024-09-20T22:29:49.356 INFO:teuthology.orchestra.run.smithi037.stdout:node-exporter.smithi037 smithi037 *:9100 running (9m) 2m ago 18m 15.4M - 1.5.0 0da6a335fe13 433e6aaf6137 2024-09-20T22:29:49.356 INFO:teuthology.orchestra.run.smithi037.stdout:node-exporter.smithi194 smithi194 *:9100 running (9m) 28s ago 17m 15.8M - 1.5.0 0da6a335fe13 1d30bb810ad1 2024-09-20T22:29:49.356 INFO:teuthology.orchestra.run.smithi037.stdout:osd.0 smithi037 running (5m) 2m ago 17m 367M 1937M 19.1.1-367-g9e2ac70b cc9251b744a3 c5568bd43c07 2024-09-20T22:29:49.357 INFO:teuthology.orchestra.run.smithi037.stdout:osd.1 smithi037 running (4m) 2m ago 16m 269M 1937M 19.1.1-367-g9e2ac70b cc9251b744a3 f6b715391469 2024-09-20T22:29:49.357 INFO:teuthology.orchestra.run.smithi037.stdout:osd.2 smithi037 running (3m) 2m ago 16m 211M 1937M 19.1.1-367-g9e2ac70b cc9251b744a3 0922768db2a2 2024-09-20T22:29:49.357 INFO:teuthology.orchestra.run.smithi037.stdout:osd.3 smithi037 running (3m) 2m ago 15m 12.4M 1937M 19.1.1-367-g9e2ac70b cc9251b744a3 312635406c2a 2024-09-20T22:29:49.357 INFO:teuthology.orchestra.run.smithi037.stdout:osd.4 smithi194 running (2m) 28s ago 15m 403M 2704M 19.1.1-367-g9e2ac70b cc9251b744a3 641138d63e50 2024-09-20T22:29:49.357 INFO:teuthology.orchestra.run.smithi037.stdout:osd.5 smithi194 running (106s) 28s ago 14m 299M 2704M 19.1.1-367-g9e2ac70b cc9251b744a3 9a2a44a8db21 2024-09-20T22:29:49.357 INFO:teuthology.orchestra.run.smithi037.stdout:osd.6 smithi194 running (69s) 28s ago 14m 260M 2704M 19.1.1-367-g9e2ac70b cc9251b744a3 2daa9faae9a2 2024-09-20T22:29:49.357 INFO:teuthology.orchestra.run.smithi037.stdout:osd.7 smithi194 running (32s) 28s ago 14m 11.8M 2704M 19.1.1-367-g9e2ac70b cc9251b744a3 281a2861e7e6 2024-09-20T22:29:49.357 INFO:teuthology.orchestra.run.smithi037.stdout:prometheus.smithi037 smithi037 *:9095 running (8m) 2m ago 17m 66.0M - 2.43.0 a07b618ecd1d 05b826dde2ff 2024-09-20T22:29:49.376 INFO:teuthology.orchestra.run.smithi037.stdout: 5 cleanup 5 sec 2024-09-20T22:29:49.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:49 smithi194 ceph-mon[51725]: pgmap v312: 97 pgs: 97 active+clean; 321 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 6.2 MiB/s wr, 262 op/s; 175 KiB/s, 11 objects/s recovering 2024-09-20T22:29:49.789 INFO:teuthology.orchestra.run.smithi037.stdout:{ 2024-09-20T22:29:49.789 INFO:teuthology.orchestra.run.smithi037.stdout: "mon": { 2024-09-20T22:29:49.789 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 2 2024-09-20T22:29:49.789 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:29:49.790 INFO:teuthology.orchestra.run.smithi037.stdout: "mgr": { 2024-09-20T22:29:49.790 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 2 2024-09-20T22:29:49.790 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:29:49.790 INFO:teuthology.orchestra.run.smithi037.stdout: "osd": { 2024-09-20T22:29:49.790 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 8 2024-09-20T22:29:49.790 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:29:49.790 INFO:teuthology.orchestra.run.smithi037.stdout: "mds": { 2024-09-20T22:29:49.790 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-20T22:29:49.790 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 1 2024-09-20T22:29:49.790 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:29:49.790 INFO:teuthology.orchestra.run.smithi037.stdout: "overall": { 2024-09-20T22:29:49.790 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-20T22:29:49.790 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 13 2024-09-20T22:29:49.790 INFO:teuthology.orchestra.run.smithi037.stdout: } 2024-09-20T22:29:49.791 INFO:teuthology.orchestra.run.smithi037.stdout:} 2024-09-20T22:29:50.205 INFO:teuthology.orchestra.run.smithi037.stdout: 0 cleanup 6 sec 2024-09-20T22:29:50.206 INFO:teuthology.orchestra.run.smithi037.stdout: 2024-09-20T22:29:50.206 INFO:teuthology.orchestra.run.smithi037.stdout: Operation Count AvgLat MaxLat 2024-09-20T22:29:50.206 INFO:teuthology.orchestra.run.smithi037.stdout: ---------------------------------------- 2024-09-20T22:29:50.206 INFO:teuthology.orchestra.run.smithi037.stdout: NTCreateX 312 40.093 150.763 2024-09-20T22:29:50.206 INFO:teuthology.orchestra.run.smithi037.stdout: Close 302 19.806 161.894 2024-09-20T22:29:50.206 INFO:teuthology.orchestra.run.smithi037.stdout: Unlink 3 19.290 31.567 2024-09-20T22:29:50.206 INFO:teuthology.orchestra.run.smithi037.stdout: Qpathinfo 18 5.848 55.000 2024-09-20T22:29:50.206 INFO:teuthology.orchestra.run.smithi037.stdout: Qfileinfo 306 19.416 57.679 2024-09-20T22:29:50.206 INFO:teuthology.orchestra.run.smithi037.stdout: Qfsinfo 31 0.553 3.149 2024-09-20T22:29:50.206 INFO:teuthology.orchestra.run.smithi037.stdout: Sfileinfo 11 20.145 38.639 2024-09-20T22:29:50.206 INFO:teuthology.orchestra.run.smithi037.stdout: Find 28 0.945 2.301 2024-09-20T22:29:50.206 INFO:teuthology.orchestra.run.smithi037.stdout: WriteX 1459 0.030 0.117 2024-09-20T22:29:50.206 INFO:teuthology.orchestra.run.smithi037.stdout: ReadX 117 0.016 0.748 2024-09-20T22:29:50.207 INFO:teuthology.orchestra.run.smithi037.stdout: Flush 12 24.091 50.600 2024-09-20T22:29:50.207 INFO:teuthology.orchestra.run.smithi037.stdout: 2024-09-20T22:29:50.207 INFO:teuthology.orchestra.run.smithi037.stdout:Throughput 12.2892 MB/sec 5 clients 5 procs max_latency=327.907 ms 2024-09-20T22:29:50.208 DEBUG:teuthology.parallel:result is None 2024-09-20T22:29:50.307 INFO:teuthology.orchestra.run.smithi037.stdout:{ 2024-09-20T22:29:50.308 INFO:teuthology.orchestra.run.smithi037.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5", 2024-09-20T22:29:50.308 INFO:teuthology.orchestra.run.smithi037.stdout: "in_progress": true, 2024-09-20T22:29:50.308 INFO:teuthology.orchestra.run.smithi037.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-20T22:29:50.308 INFO:teuthology.orchestra.run.smithi037.stdout: "services_complete": [ 2024-09-20T22:29:50.308 INFO:teuthology.orchestra.run.smithi037.stdout: "mon", 2024-09-20T22:29:50.308 INFO:teuthology.orchestra.run.smithi037.stdout: "mgr", 2024-09-20T22:29:50.308 INFO:teuthology.orchestra.run.smithi037.stdout: "crash", 2024-09-20T22:29:50.308 INFO:teuthology.orchestra.run.smithi037.stdout: "osd" 2024-09-20T22:29:50.308 INFO:teuthology.orchestra.run.smithi037.stdout: ], 2024-09-20T22:29:50.308 INFO:teuthology.orchestra.run.smithi037.stdout: "progress": "14/23 daemons upgraded", 2024-09-20T22:29:50.308 INFO:teuthology.orchestra.run.smithi037.stdout: "message": "Currently upgrading mds daemons", 2024-09-20T22:29:50.309 INFO:teuthology.orchestra.run.smithi037.stdout: "is_paused": false 2024-09-20T22:29:50.309 INFO:teuthology.orchestra.run.smithi037.stdout:} 2024-09-20T22:29:50.369 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:50 smithi037 ceph-mon[65640]: from='client.34462 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:29:50.369 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:50 smithi037 ceph-mon[65640]: from='client.44383 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:29:50.369 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:50 smithi037 ceph-mon[65640]: from='client.? 172.21.15.37:0/2509630976' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:29:50.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:50 smithi194 ceph-mon[51725]: from='client.34462 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:29:50.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:50 smithi194 ceph-mon[51725]: from='client.44383 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:29:50.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:50 smithi194 ceph-mon[51725]: from='client.? 172.21.15.37:0/2509630976' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:29:50.741 INFO:teuthology.orchestra.run.smithi037.stdout:HEALTH_OK 2024-09-20T22:29:51.540 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:51 smithi037 ceph-mon[65640]: from='client.34470 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:29:51.540 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:51 smithi037 ceph-mon[65640]: pgmap v313: 97 pgs: 97 active+clean; 311 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 7.2 MiB/s wr, 276 op/s; 175 KiB/s, 11 objects/s recovering 2024-09-20T22:29:51.540 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:51 smithi037 ceph-mon[65640]: from='client.? 172.21.15.37:0/459921897' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T22:29:51.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:51 smithi194 ceph-mon[51725]: from='client.34470 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:29:51.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:51 smithi194 ceph-mon[51725]: pgmap v313: 97 pgs: 97 active+clean; 311 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 7.2 MiB/s wr, 276 op/s; 175 KiB/s, 11 objects/s recovering 2024-09-20T22:29:51.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:51 smithi194 ceph-mon[51725]: from='client.? 172.21.15.37:0/459921897' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T22:29:52.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:52 smithi194 ceph-mon[51725]: from='client.34478 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:29:52.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:52 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:29:52.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:52 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:29:52.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:52 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:29:52.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:52 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:29:52.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:52 smithi037 ceph-mon[65640]: from='client.34478 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:29:52.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:52 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:29:52.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:52 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:29:52.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:52 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:29:52.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:52 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:29:53.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:53 smithi037 ceph-mon[65640]: pgmap v314: 97 pgs: 97 active+clean; 320 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 33 KiB/s rd, 10 MiB/s wr, 293 op/s 2024-09-20T22:29:53.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:53 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:29:53.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:53 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:29:53.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:53 smithi194 ceph-mon[51725]: pgmap v314: 97 pgs: 97 active+clean; 320 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 33 KiB/s rd, 10 MiB/s wr, 293 op/s 2024-09-20T22:29:53.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:53 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:29:53.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:53 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:29:55.442 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:55 smithi037 ceph-mon[65640]: pgmap v315: 97 pgs: 97 active+clean; 331 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 28 KiB/s rd, 11 MiB/s wr, 298 op/s 2024-09-20T22:29:55.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:55 smithi194 ceph-mon[51725]: pgmap v315: 97 pgs: 97 active+clean; 331 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 28 KiB/s rd, 11 MiB/s wr, 298 op/s 2024-09-20T22:29:57.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:57 smithi194 ceph-mon[51725]: pgmap v316: 97 pgs: 97 active+clean; 277 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 18 KiB/s rd, 9.7 MiB/s wr, 284 op/s 2024-09-20T22:29:57.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:57 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:29:57.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:57 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:29:57.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:57 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:29:57.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:57 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:29:57.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:57 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:29:57.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:57 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:29:57.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:57 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:29:57.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:57 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:29:57.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:57 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:29:57.596 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:57 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:29:57.596 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:57 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:29:57.596 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:57 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi194.lkroso"]}]: dispatch 2024-09-20T22:29:57.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:57 smithi037 ceph-mon[65640]: pgmap v316: 97 pgs: 97 active+clean; 277 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 18 KiB/s rd, 9.7 MiB/s wr, 284 op/s 2024-09-20T22:29:57.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:57 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:29:57.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:57 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:29:57.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:57 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:29:57.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:57 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:29:57.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:57 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:29:57.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:57 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:29:57.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:57 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:29:57.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:57 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:29:57.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:57 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:29:57.664 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:57 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:29:57.664 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:57 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:29:57.664 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:57 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi194.lkroso"]}]: dispatch 2024-09-20T22:29:58.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:58 smithi194 ceph-mon[51725]: Detected new or changed devices on smithi037 2024-09-20T22:29:58.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:58 smithi194 ceph-mon[51725]: Upgrade: It appears safe to stop mds.foofs.smithi194.lkroso 2024-09-20T22:29:58.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:58 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:29:58.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:58 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi194.lkroso", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-20T22:29:58.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:58 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:29:58.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:58 smithi037 ceph-mon[65640]: Detected new or changed devices on smithi037 2024-09-20T22:29:58.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:58 smithi037 ceph-mon[65640]: Upgrade: It appears safe to stop mds.foofs.smithi194.lkroso 2024-09-20T22:29:58.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:58 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:29:58.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:58 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi194.lkroso", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-20T22:29:58.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:58 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:29:59.240 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:59 smithi194 ceph-mon[51725]: Upgrade: Updating mds.foofs.smithi194.lkroso 2024-09-20T22:29:59.241 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:59 smithi194 ceph-mon[51725]: Deploying daemon mds.foofs.smithi194.lkroso on smithi194 2024-09-20T22:29:59.241 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:29:59 smithi194 ceph-mon[51725]: pgmap v317: 97 pgs: 97 active+clean; 230 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 7.6 MiB/s wr, 240 op/s 2024-09-20T22:29:59.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:59 smithi037 ceph-mon[65640]: Upgrade: Updating mds.foofs.smithi194.lkroso 2024-09-20T22:29:59.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:59 smithi037 ceph-mon[65640]: Deploying daemon mds.foofs.smithi194.lkroso on smithi194 2024-09-20T22:29:59.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:59 smithi037 ceph-mon[65640]: pgmap v317: 97 pgs: 97 active+clean; 230 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 7.6 MiB/s wr, 240 op/s 2024-09-20T22:30:00.095 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:29:59 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-5[58907]: 2024-09-20T22:29:59.810+0000 7f63d17bc640 -1 bdev(0x5599db6a6c00 /var/lib/ceph/osd/ceph-5/block) aio_submit retries 5 2024-09-20T22:30:00.095 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:29:59 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-5[58907]: 2024-09-20T22:29:59.812+0000 7f63d27be640 -1 bdev(0x5599db6a6c00 /var/lib/ceph/osd/ceph-5/block) aio_submit retries 8 2024-09-20T22:30:00.095 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:29:59 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:29:59.810+0000 7f26130f2640 -1 bdev(0x564dcee7ec00 /var/lib/ceph/osd/ceph-7/block) aio_submit retries 3 2024-09-20T22:30:00.095 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:29:59 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:29:59.814+0000 7f26198ff640 -1 bdev(0x564dcee7ec00 /var/lib/ceph/osd/ceph-7/block) aio_submit retries 10 2024-09-20T22:30:00.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:29:59 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-mon-smithi037[65636]: 2024-09-20T22:29:59.786+0000 7f97ffa2b640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-20T22:30:00.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:00 smithi194 ceph-mon[51725]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-20T22:30:00.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:00 smithi194 ceph-mon[51725]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-20T22:30:00.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:00 smithi194 ceph-mon[51725]: osdmap e160: 8 total, 8 up, 8 in 2024-09-20T22:30:00.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:00 smithi194 ceph-mon[51725]: Standby daemon mds.foofs.smithi037.sdtedi assigned to filesystem foofs as rank 0 2024-09-20T22:30:00.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:00 smithi194 ceph-mon[51725]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-09-20T22:30:00.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:00 smithi194 ceph-mon[51725]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-20T22:30:00.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:00 smithi194 ceph-mon[51725]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-09-20T22:30:00.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:00 smithi194 ceph-mon[51725]: fsmap foofs:1/1 {0=foofs.smithi037.sdtedi=up:replay} 2024-09-20T22:30:00.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:00 smithi194 ceph-mon[51725]: Health detail: HEALTH_WARN 1 filesystem is degraded; insufficient standby MDS daemons available 2024-09-20T22:30:00.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:00 smithi194 ceph-mon[51725]: [WRN] FS_DEGRADED: 1 filesystem is degraded 2024-09-20T22:30:00.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:00 smithi194 ceph-mon[51725]: fs foofs is degraded 2024-09-20T22:30:00.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:00 smithi194 ceph-mon[51725]: [WRN] MDS_INSUFFICIENT_STANDBY: insufficient standby MDS daemons available 2024-09-20T22:30:00.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:00 smithi194 ceph-mon[51725]: have 0; want 1 more 2024-09-20T22:30:00.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:00 smithi037 ceph-mon[65640]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-20T22:30:00.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:00 smithi037 ceph-mon[65640]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-20T22:30:00.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:00 smithi037 ceph-mon[65640]: osdmap e160: 8 total, 8 up, 8 in 2024-09-20T22:30:00.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:00 smithi037 ceph-mon[65640]: Standby daemon mds.foofs.smithi037.sdtedi assigned to filesystem foofs as rank 0 2024-09-20T22:30:00.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:00 smithi037 ceph-mon[65640]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-09-20T22:30:00.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:00 smithi037 ceph-mon[65640]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-20T22:30:00.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:00 smithi037 ceph-mon[65640]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-09-20T22:30:00.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:00 smithi037 ceph-mon[65640]: fsmap foofs:1/1 {0=foofs.smithi037.sdtedi=up:replay} 2024-09-20T22:30:00.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:00 smithi037 ceph-mon[65640]: Health detail: HEALTH_WARN 1 filesystem is degraded; insufficient standby MDS daemons available 2024-09-20T22:30:00.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:00 smithi037 ceph-mon[65640]: [WRN] FS_DEGRADED: 1 filesystem is degraded 2024-09-20T22:30:00.664 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:00 smithi037 ceph-mon[65640]: fs foofs is degraded 2024-09-20T22:30:00.664 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:00 smithi037 ceph-mon[65640]: [WRN] MDS_INSUFFICIENT_STANDBY: insufficient standby MDS daemons available 2024-09-20T22:30:00.664 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:00 smithi037 ceph-mon[65640]: have 0; want 1 more 2024-09-20T22:30:01.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:01 smithi194 ceph-mon[51725]: pgmap v319: 97 pgs: 97 active+clean; 230 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.5 KiB/s rd, 4.6 MiB/s wr, 129 op/s 2024-09-20T22:30:01.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:01 smithi037 ceph-mon[65640]: pgmap v319: 97 pgs: 97 active+clean; 230 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.5 KiB/s rd, 4.6 MiB/s wr, 129 op/s 2024-09-20T22:30:02.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:02 smithi194 ceph-mon[51725]: reconnect by client.14550 172.21.15.37:0/859528995 after 0.000999987 2024-09-20T22:30:02.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:02 smithi194 ceph-mon[51725]: reconnect by client.24363 172.21.15.194:0/2491128762 after 0.000999987 2024-09-20T22:30:02.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:02 smithi194 ceph-mon[51725]: mds.? [v2:172.21.15.37:6834/1667079263,v1:172.21.15.37:6835/1667079263] up:reconnect 2024-09-20T22:30:02.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:02 smithi194 ceph-mon[51725]: fsmap foofs:1/1 {0=foofs.smithi037.sdtedi=up:reconnect} 2024-09-20T22:30:02.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:02 smithi037 ceph-mon[65640]: reconnect by client.14550 172.21.15.37:0/859528995 after 0.000999987 2024-09-20T22:30:02.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:02 smithi037 ceph-mon[65640]: reconnect by client.24363 172.21.15.194:0/2491128762 after 0.000999987 2024-09-20T22:30:02.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:02 smithi037 ceph-mon[65640]: mds.? [v2:172.21.15.37:6834/1667079263,v1:172.21.15.37:6835/1667079263] up:reconnect 2024-09-20T22:30:02.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:02 smithi037 ceph-mon[65640]: fsmap foofs:1/1 {0=foofs.smithi037.sdtedi=up:reconnect} 2024-09-20T22:30:03.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:03 smithi194 ceph-mon[51725]: pgmap v320: 97 pgs: 97 active+clean; 230 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.4 MiB/s rd, 1.4 MiB/s wr, 103 op/s 2024-09-20T22:30:03.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:03 smithi194 ceph-mon[51725]: mds.? [v2:172.21.15.37:6834/1667079263,v1:172.21.15.37:6835/1667079263] up:rejoin 2024-09-20T22:30:03.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:03 smithi194 ceph-mon[51725]: fsmap foofs:1/1 {0=foofs.smithi037.sdtedi=up:rejoin} 2024-09-20T22:30:03.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:03 smithi194 ceph-mon[51725]: daemon mds.foofs.smithi037.sdtedi is now active in filesystem foofs as rank 0 2024-09-20T22:30:03.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:03 smithi037 ceph-mon[65640]: pgmap v320: 97 pgs: 97 active+clean; 230 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.4 MiB/s rd, 1.4 MiB/s wr, 103 op/s 2024-09-20T22:30:03.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:03 smithi037 ceph-mon[65640]: mds.? [v2:172.21.15.37:6834/1667079263,v1:172.21.15.37:6835/1667079263] up:rejoin 2024-09-20T22:30:03.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:03 smithi037 ceph-mon[65640]: fsmap foofs:1/1 {0=foofs.smithi037.sdtedi=up:rejoin} 2024-09-20T22:30:03.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:03 smithi037 ceph-mon[65640]: daemon mds.foofs.smithi037.sdtedi is now active in filesystem foofs as rank 0 2024-09-20T22:30:04.437 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:04 smithi194 ceph-mon[51725]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-20T22:30:04.438 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:04 smithi194 ceph-mon[51725]: mds.? [v2:172.21.15.37:6834/1667079263,v1:172.21.15.37:6835/1667079263] up:active 2024-09-20T22:30:04.438 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:04 smithi194 ceph-mon[51725]: fsmap foofs:1 {0=foofs.smithi037.sdtedi=up:active} 2024-09-20T22:30:04.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:04 smithi037 ceph-mon[65640]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-20T22:30:04.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:04 smithi037 ceph-mon[65640]: mds.? [v2:172.21.15.37:6834/1667079263,v1:172.21.15.37:6835/1667079263] up:active 2024-09-20T22:30:04.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:04 smithi037 ceph-mon[65640]: fsmap foofs:1 {0=foofs.smithi037.sdtedi=up:active} 2024-09-20T22:30:05.485 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:05 smithi194 ceph-mon[51725]: pgmap v321: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 39 KiB/s wr, 38 op/s 2024-09-20T22:30:05.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:05 smithi037 ceph-mon[65640]: pgmap v321: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 39 KiB/s wr, 38 op/s 2024-09-20T22:30:07.189 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:06 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:07.189 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:06 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:07.189 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:06 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:30:07.189 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:06 smithi194 ceph-mon[51725]: pgmap v322: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 39 KiB/s wr, 25 op/s 2024-09-20T22:30:07.189 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:06 smithi194 ceph-mon[51725]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-09-20T22:30:07.189 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:06 smithi194 ceph-mon[51725]: Cluster is now healthy 2024-09-20T22:30:07.189 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:06 smithi194 ceph-mon[51725]: mds.? [v2:172.21.15.194:6832/3860700832,v1:172.21.15.194:6833/3860700832] up:boot 2024-09-20T22:30:07.189 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:06 smithi194 ceph-mon[51725]: fsmap foofs:1 {0=foofs.smithi037.sdtedi=up:active} 1 up:standby 2024-09-20T22:30:07.189 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:06 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi194.lkroso"}]: dispatch 2024-09-20T22:30:07.189 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:06 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:30:07.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:06 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:07.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:06 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:07.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:06 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:30:07.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:06 smithi037 ceph-mon[65640]: pgmap v322: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 39 KiB/s wr, 25 op/s 2024-09-20T22:30:07.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:06 smithi037 ceph-mon[65640]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-09-20T22:30:07.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:06 smithi037 ceph-mon[65640]: Cluster is now healthy 2024-09-20T22:30:07.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:06 smithi037 ceph-mon[65640]: mds.? [v2:172.21.15.194:6832/3860700832,v1:172.21.15.194:6833/3860700832] up:boot 2024-09-20T22:30:07.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:06 smithi037 ceph-mon[65640]: fsmap foofs:1 {0=foofs.smithi037.sdtedi=up:active} 1 up:standby 2024-09-20T22:30:07.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:06 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi194.lkroso"}]: dispatch 2024-09-20T22:30:07.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:06 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:30:09.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:09 smithi194 ceph-mon[51725]: pgmap v323: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 11 KiB/s wr, 15 op/s 2024-09-20T22:30:09.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:09 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:09.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:09 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:09.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:09 smithi037 ceph-mon[65640]: pgmap v323: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 11 KiB/s wr, 15 op/s 2024-09-20T22:30:09.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:09 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:09.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:09 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:10.550 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:10 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:10.550 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:10 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:10.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:10 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:10.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:10 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:11.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:11 smithi194 ceph-mon[51725]: pgmap v324: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 10 KiB/s wr, 15 op/s 2024-09-20T22:30:11.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:11 smithi037 ceph-mon[65640]: pgmap v324: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 10 KiB/s wr, 15 op/s 2024-09-20T22:30:13.390 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:13 smithi037 ceph-mon[65640]: pgmap v325: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 8.8 KiB/s wr, 12 op/s 2024-09-20T22:30:13.390 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:13 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:13.390 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:13 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:13.391 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:13 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:30:13.391 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:13 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:30:13.391 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:13 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:13.391 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:13 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:13.391 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:13 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:30:13.391 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:13 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:13.391 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:13 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:13.391 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:13 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:13.391 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:13 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:13.391 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:13 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:13.391 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:13 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:13.391 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:13 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi037.sdtedi"}]: dispatch 2024-09-20T22:30:13.391 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:13 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi037.sdtedi"}]': finished 2024-09-20T22:30:13.391 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:13 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi194.lkroso"}]: dispatch 2024-09-20T22:30:13.392 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:13 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi194.lkroso"}]': finished 2024-09-20T22:30:13.392 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:13 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:13.392 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:13 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:13.392 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:13 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:13.392 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:13 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:13.392 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:13 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:13.392 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:13 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:13.392 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:13 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:13.392 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:13 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:13.392 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:13 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:13.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:13 smithi194 ceph-mon[51725]: pgmap v325: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 8.8 KiB/s wr, 12 op/s 2024-09-20T22:30:13.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:13 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:13.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:13 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:13.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:13 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:30:13.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:13 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:30:13.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:13 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:13.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:13 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:13.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:13 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:30:13.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:13 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:13.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:13 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:13.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:13 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:13.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:13 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:13.596 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:13 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:13.596 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:13 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:13.596 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:13 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi037.sdtedi"}]: dispatch 2024-09-20T22:30:13.596 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:13 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi037.sdtedi"}]': finished 2024-09-20T22:30:13.596 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:13 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi194.lkroso"}]: dispatch 2024-09-20T22:30:13.596 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:13 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi194.lkroso"}]': finished 2024-09-20T22:30:13.596 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:13 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:13.596 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:13 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:13.596 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:13 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:13.596 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:13 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:13.596 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:13 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:13.596 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:13 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:13.596 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:13 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:13.596 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:13 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:13.597 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:13 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:14.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:14 smithi194 ceph-mon[51725]: Detected new or changed devices on smithi194 2024-09-20T22:30:14.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:14 smithi194 ceph-mon[51725]: Upgrade: Setting container_image for all mds 2024-09-20T22:30:14.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:14 smithi194 ceph-mon[51725]: Upgrade: Setting container_image for all rgw 2024-09-20T22:30:14.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:14 smithi194 ceph-mon[51725]: Upgrade: Setting container_image for all rbd-mirror 2024-09-20T22:30:14.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:14 smithi194 ceph-mon[51725]: Upgrade: Setting container_image for all ceph-exporter 2024-09-20T22:30:14.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:14 smithi194 ceph-mon[51725]: Upgrade: Setting container_image for all iscsi 2024-09-20T22:30:14.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:14 smithi037 ceph-mon[65640]: Detected new or changed devices on smithi194 2024-09-20T22:30:14.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:14 smithi037 ceph-mon[65640]: Upgrade: Setting container_image for all mds 2024-09-20T22:30:14.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:14 smithi037 ceph-mon[65640]: Upgrade: Setting container_image for all rgw 2024-09-20T22:30:14.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:14 smithi037 ceph-mon[65640]: Upgrade: Setting container_image for all rbd-mirror 2024-09-20T22:30:14.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:14 smithi037 ceph-mon[65640]: Upgrade: Setting container_image for all ceph-exporter 2024-09-20T22:30:14.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:14 smithi037 ceph-mon[65640]: Upgrade: Setting container_image for all iscsi 2024-09-20T22:30:15.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:15 smithi037 ceph-mon[65640]: pgmap v326: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 8.8 KiB/s wr, 9 op/s 2024-09-20T22:30:15.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:15 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:15.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:15 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi037.spabdc", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-20T22:30:15.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:15 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-20T22:30:15.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:15 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-20T22:30:15.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:15 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:30:15.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:15 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-20T22:30:15.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:15 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-20T22:30:15.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:15 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi037.spabdc-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-20T22:30:15.414 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:15 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:30:15.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:15 smithi194 ceph-mon[51725]: pgmap v326: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 8.8 KiB/s wr, 9 op/s 2024-09-20T22:30:15.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:15 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:15.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:15 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi037.spabdc", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-20T22:30:15.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:15 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-20T22:30:15.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:15 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-20T22:30:15.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:15 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:30:15.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:15 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-20T22:30:15.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:15 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-20T22:30:15.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:15 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi037.spabdc-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-20T22:30:15.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:15 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:30:16.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:16 smithi194 ceph-mon[51725]: Upgrade: Updating nfs.foo.0.0.smithi037.spabdc (1/2) 2024-09-20T22:30:16.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:16 smithi194 ceph-mon[51725]: Creating key for client.nfs.foo.0.0.smithi037.spabdc 2024-09-20T22:30:16.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:16 smithi194 ceph-mon[51725]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-09-20T22:30:16.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:16 smithi194 ceph-mon[51725]: Rados config object exists: conf-nfs.foo 2024-09-20T22:30:16.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:16 smithi194 ceph-mon[51725]: Creating key for client.nfs.foo.0.0.smithi037.spabdc-rgw 2024-09-20T22:30:16.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:16 smithi194 ceph-mon[51725]: Bind address in nfs.foo.0.0.smithi037.spabdc's ganesha conf is defaulting to empty 2024-09-20T22:30:16.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:16 smithi194 ceph-mon[51725]: Deploying daemon nfs.foo.0.0.smithi037.spabdc on smithi037 2024-09-20T22:30:16.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:16 smithi037 ceph-mon[65640]: Upgrade: Updating nfs.foo.0.0.smithi037.spabdc (1/2) 2024-09-20T22:30:16.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:16 smithi037 ceph-mon[65640]: Creating key for client.nfs.foo.0.0.smithi037.spabdc 2024-09-20T22:30:16.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:16 smithi037 ceph-mon[65640]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-09-20T22:30:16.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:16 smithi037 ceph-mon[65640]: Rados config object exists: conf-nfs.foo 2024-09-20T22:30:16.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:16 smithi037 ceph-mon[65640]: Creating key for client.nfs.foo.0.0.smithi037.spabdc-rgw 2024-09-20T22:30:16.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:16 smithi037 ceph-mon[65640]: Bind address in nfs.foo.0.0.smithi037.spabdc's ganesha conf is defaulting to empty 2024-09-20T22:30:16.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:16 smithi037 ceph-mon[65640]: Deploying daemon nfs.foo.0.0.smithi037.spabdc on smithi037 2024-09-20T22:30:17.498 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:17 smithi037 ceph-mon[65640]: pgmap v327: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.0 MiB/s rd, 7.2 KiB/s wr, 7 op/s 2024-09-20T22:30:17.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:17 smithi194 ceph-mon[51725]: pgmap v327: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.0 MiB/s rd, 7.2 KiB/s wr, 7 op/s 2024-09-20T22:30:18.289 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:18 smithi194 ceph-mon[51725]: pgmap v328: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.9 KiB/s rd, 7.4 KiB/s wr, 5 op/s 2024-09-20T22:30:18.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:18 smithi037 ceph-mon[65640]: pgmap v328: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.9 KiB/s rd, 7.4 KiB/s wr, 5 op/s 2024-09-20T22:30:21.125 INFO:teuthology.orchestra.run.smithi037.stdout:true 2024-09-20T22:30:21.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:21 smithi194 ceph-mon[51725]: pgmap v329: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 170 B/s wr, 0 op/s 2024-09-20T22:30:21.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:21 smithi194 ceph-mon[51725]: from='mds.? [v2:172.21.15.37:6834/1667079263,v1:172.21.15.37:6835/1667079263]' entity='mds.foofs.smithi037.sdtedi' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.37:0/859528995"}]: dispatch 2024-09-20T22:30:21.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:21 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:21.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:21 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:21.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:21 smithi037 ceph-mon[65640]: pgmap v329: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 170 B/s wr, 0 op/s 2024-09-20T22:30:21.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:21 smithi037 ceph-mon[65640]: from='mds.? [v2:172.21.15.37:6834/1667079263,v1:172.21.15.37:6835/1667079263]' entity='mds.foofs.smithi037.sdtedi' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.37:0/859528995"}]: dispatch 2024-09-20T22:30:21.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:21 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:21.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:21 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:21.816 INFO:teuthology.orchestra.run.smithi037.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T22:30:21.816 INFO:teuthology.orchestra.run.smithi037.stdout:alertmanager.smithi037 smithi037 *:9093,9094 running (10m) 29s ago 19m 26.1M - 0.25.0 c8568f914cd2 d1f99b3cd707 2024-09-20T22:30:21.816 INFO:teuthology.orchestra.run.smithi037.stdout:crash.smithi037 smithi037 running (6m) 29s ago 19m 6895k - 19.1.1-367-g9e2ac70b cc9251b744a3 77e510d8bdb1 2024-09-20T22:30:21.816 INFO:teuthology.orchestra.run.smithi037.stdout:crash.smithi194 smithi194 running (6m) 13s ago 18m 6895k - 19.1.1-367-g9e2ac70b cc9251b744a3 4d169d1309bc 2024-09-20T22:30:21.816 INFO:teuthology.orchestra.run.smithi037.stdout:grafana.smithi037 smithi037 *:3000 running (10m) 29s ago 18m 53.5M - dad864ee21e9 2306f3c2da19 2024-09-20T22:30:21.816 INFO:teuthology.orchestra.run.smithi037.stdout:mds.foofs.smithi037.sdtedi smithi037 running (36s) 29s ago 13m 11.0M - 19.1.1-367-g9e2ac70b cc9251b744a3 85193a675951 2024-09-20T22:30:21.817 INFO:teuthology.orchestra.run.smithi037.stdout:mds.foofs.smithi194.lkroso smithi194 running (16s) 13s ago 13m 10.7M - 19.1.1-367-g9e2ac70b cc9251b744a3 4e77a613e3bc 2024-09-20T22:30:21.817 INFO:teuthology.orchestra.run.smithi037.stdout:mgr.smithi037.lykblh smithi037 *:8443,9283,8765 running (9m) 29s ago 20m 509M - 19.1.1-367-g9e2ac70b cc9251b744a3 181e46fd9ba6 2024-09-20T22:30:21.817 INFO:teuthology.orchestra.run.smithi037.stdout:mgr.smithi194.bkcglt smithi194 *:8443,9283,8765 running (8m) 13s ago 18m 444M - 19.1.1-367-g9e2ac70b cc9251b744a3 1fa610c97517 2024-09-20T22:30:21.817 INFO:teuthology.orchestra.run.smithi037.stdout:mon.smithi037 smithi037 running (8m) 29s ago 20m 60.9M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 c0229eceb527 2024-09-20T22:30:21.817 INFO:teuthology.orchestra.run.smithi037.stdout:mon.smithi194 smithi194 running (7m) 13s ago 18m 50.1M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 97854807ba91 2024-09-20T22:30:21.817 INFO:teuthology.orchestra.run.smithi037.stdout:nfs.foo.0.0.smithi037.spabdc smithi037 *:2049 starting - - - - 2024-09-20T22:30:21.818 INFO:teuthology.orchestra.run.smithi037.stdout:nfs.foo.1.0.smithi194.qzjyej smithi194 *:2049 running (12m) 13s ago 12m 90.5M - 4.0 e1d6a67b021e c07fe0eb7c75 2024-09-20T22:30:21.818 INFO:teuthology.orchestra.run.smithi037.stdout:node-exporter.smithi037 smithi037 *:9100 running (10m) 29s ago 18m 15.4M - 1.5.0 0da6a335fe13 433e6aaf6137 2024-09-20T22:30:21.818 INFO:teuthology.orchestra.run.smithi037.stdout:node-exporter.smithi194 smithi194 *:9100 running (9m) 13s ago 18m 16.2M - 1.5.0 0da6a335fe13 1d30bb810ad1 2024-09-20T22:30:21.818 INFO:teuthology.orchestra.run.smithi037.stdout:osd.0 smithi037 running (6m) 29s ago 17m 405M 1937M 19.1.1-367-g9e2ac70b cc9251b744a3 c5568bd43c07 2024-09-20T22:30:21.818 INFO:teuthology.orchestra.run.smithi037.stdout:osd.1 smithi037 running (5m) 29s ago 17m 328M 1937M 19.1.1-367-g9e2ac70b cc9251b744a3 f6b715391469 2024-09-20T22:30:21.818 INFO:teuthology.orchestra.run.smithi037.stdout:osd.2 smithi037 running (4m) 29s ago 16m 297M 1937M 19.1.1-367-g9e2ac70b cc9251b744a3 0922768db2a2 2024-09-20T22:30:21.818 INFO:teuthology.orchestra.run.smithi037.stdout:osd.3 smithi037 running (3m) 29s ago 16m 399M 1937M 19.1.1-367-g9e2ac70b cc9251b744a3 312635406c2a 2024-09-20T22:30:21.818 INFO:teuthology.orchestra.run.smithi037.stdout:osd.4 smithi194 running (2m) 13s ago 15m 428M 2704M 19.1.1-367-g9e2ac70b cc9251b744a3 641138d63e50 2024-09-20T22:30:21.819 INFO:teuthology.orchestra.run.smithi037.stdout:osd.5 smithi194 running (2m) 13s ago 15m 335M 2704M 19.1.1-367-g9e2ac70b cc9251b744a3 9a2a44a8db21 2024-09-20T22:30:21.819 INFO:teuthology.orchestra.run.smithi037.stdout:osd.6 smithi194 running (101s) 13s ago 15m 293M 2704M 19.1.1-367-g9e2ac70b cc9251b744a3 2daa9faae9a2 2024-09-20T22:30:21.819 INFO:teuthology.orchestra.run.smithi037.stdout:osd.7 smithi194 running (65s) 13s ago 14m 280M 2704M 19.1.1-367-g9e2ac70b cc9251b744a3 281a2861e7e6 2024-09-20T22:30:21.819 INFO:teuthology.orchestra.run.smithi037.stdout:prometheus.smithi037 smithi037 *:9095 running (8m) 29s ago 18m 66.2M - 2.43.0 a07b618ecd1d 05b826dde2ff 2024-09-20T22:30:22.121 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:22 smithi194 ceph-mon[51725]: Evicting (and blocklisting) client session 14550 (172.21.15.37:0/859528995) 2024-09-20T22:30:22.122 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:22 smithi194 ceph-mon[51725]: from='mds.? [v2:172.21.15.37:6834/1667079263,v1:172.21.15.37:6835/1667079263]' entity='mds.foofs.smithi037.sdtedi' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.37:0/859528995"}]': finished 2024-09-20T22:30:22.122 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:22 smithi194 ceph-mon[51725]: osdmap e161: 8 total, 8 up, 8 in 2024-09-20T22:30:22.122 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:22 smithi194 ceph-mon[51725]: from='client.34512 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:30:22.122 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:22 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:30:22.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:22 smithi037 ceph-mon[65640]: Evicting (and blocklisting) client session 14550 (172.21.15.37:0/859528995) 2024-09-20T22:30:22.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:22 smithi037 ceph-mon[65640]: from='mds.? [v2:172.21.15.37:6834/1667079263,v1:172.21.15.37:6835/1667079263]' entity='mds.foofs.smithi037.sdtedi' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.37:0/859528995"}]': finished 2024-09-20T22:30:22.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:22 smithi037 ceph-mon[65640]: osdmap e161: 8 total, 8 up, 8 in 2024-09-20T22:30:22.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:22 smithi037 ceph-mon[65640]: from='client.34512 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:30:22.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:22 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:30:22.239 INFO:teuthology.orchestra.run.smithi037.stdout:{ 2024-09-20T22:30:22.239 INFO:teuthology.orchestra.run.smithi037.stdout: "mon": { 2024-09-20T22:30:22.239 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 2 2024-09-20T22:30:22.239 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:30:22.239 INFO:teuthology.orchestra.run.smithi037.stdout: "mgr": { 2024-09-20T22:30:22.239 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 2 2024-09-20T22:30:22.239 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:30:22.239 INFO:teuthology.orchestra.run.smithi037.stdout: "osd": { 2024-09-20T22:30:22.240 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 8 2024-09-20T22:30:22.240 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:30:22.240 INFO:teuthology.orchestra.run.smithi037.stdout: "mds": { 2024-09-20T22:30:22.240 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 2 2024-09-20T22:30:22.240 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:30:22.240 INFO:teuthology.orchestra.run.smithi037.stdout: "overall": { 2024-09-20T22:30:22.240 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 14 2024-09-20T22:30:22.240 INFO:teuthology.orchestra.run.smithi037.stdout: } 2024-09-20T22:30:22.240 INFO:teuthology.orchestra.run.smithi037.stdout:} 2024-09-20T22:30:22.641 INFO:teuthology.orchestra.run.smithi037.stdout:{ 2024-09-20T22:30:22.641 INFO:teuthology.orchestra.run.smithi037.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5", 2024-09-20T22:30:22.641 INFO:teuthology.orchestra.run.smithi037.stdout: "in_progress": true, 2024-09-20T22:30:22.641 INFO:teuthology.orchestra.run.smithi037.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-20T22:30:22.641 INFO:teuthology.orchestra.run.smithi037.stdout: "services_complete": [ 2024-09-20T22:30:22.641 INFO:teuthology.orchestra.run.smithi037.stdout: "mds", 2024-09-20T22:30:22.641 INFO:teuthology.orchestra.run.smithi037.stdout: "crash", 2024-09-20T22:30:22.641 INFO:teuthology.orchestra.run.smithi037.stdout: "osd", 2024-09-20T22:30:22.641 INFO:teuthology.orchestra.run.smithi037.stdout: "mon", 2024-09-20T22:30:22.642 INFO:teuthology.orchestra.run.smithi037.stdout: "mgr" 2024-09-20T22:30:22.642 INFO:teuthology.orchestra.run.smithi037.stdout: ], 2024-09-20T22:30:22.642 INFO:teuthology.orchestra.run.smithi037.stdout: "progress": "16/23 daemons upgraded", 2024-09-20T22:30:22.642 INFO:teuthology.orchestra.run.smithi037.stdout: "message": "Currently upgrading nfs daemons", 2024-09-20T22:30:22.642 INFO:teuthology.orchestra.run.smithi037.stdout: "is_paused": false 2024-09-20T22:30:22.642 INFO:teuthology.orchestra.run.smithi037.stdout:} 2024-09-20T22:30:23.058 INFO:teuthology.orchestra.run.smithi037.stdout:HEALTH_OK 2024-09-20T22:30:23.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:23 smithi194 ceph-mon[51725]: from='client.34516 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:30:23.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:23 smithi194 ceph-mon[51725]: from='client.34520 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:30:23.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:23 smithi194 ceph-mon[51725]: pgmap v331: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 409 B/s rd, 204 B/s wr, 0 op/s 2024-09-20T22:30:23.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:23 smithi194 ceph-mon[51725]: Upgrade: Updating nfs.foo.1.0.smithi194.qzjyej (2/2) 2024-09-20T22:30:23.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:23 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:23.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:23 smithi194 ceph-mon[51725]: Creating key for client.nfs.foo.1.0.smithi194.qzjyej 2024-09-20T22:30:23.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:23 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi194.qzjyej", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-20T22:30:23.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:23 smithi194 ceph-mon[51725]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-09-20T22:30:23.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:23 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-20T22:30:23.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:23 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-20T22:30:23.346 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:23 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:30:23.346 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:23 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-20T22:30:23.346 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:23 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-20T22:30:23.346 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:23 smithi194 ceph-mon[51725]: from='client.? 172.21.15.37:0/2784792619' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:23.346 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:23 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi194.qzjyej-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-20T22:30:23.346 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:23 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:30:23.346 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:23 smithi194 ceph-mon[51725]: from='client.? 172.21.15.37:0/894314240' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T22:30:23.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:23 smithi037 ceph-mon[65640]: from='client.34516 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:30:23.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:23 smithi037 ceph-mon[65640]: from='client.34520 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:30:23.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:23 smithi037 ceph-mon[65640]: pgmap v331: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 409 B/s rd, 204 B/s wr, 0 op/s 2024-09-20T22:30:23.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:23 smithi037 ceph-mon[65640]: Upgrade: Updating nfs.foo.1.0.smithi194.qzjyej (2/2) 2024-09-20T22:30:23.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:23 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:23.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:23 smithi037 ceph-mon[65640]: Creating key for client.nfs.foo.1.0.smithi194.qzjyej 2024-09-20T22:30:23.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:23 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi194.qzjyej", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-20T22:30:23.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:23 smithi037 ceph-mon[65640]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-09-20T22:30:23.414 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:23 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-20T22:30:23.414 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:23 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-20T22:30:23.414 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:23 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:30:23.414 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:23 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-20T22:30:23.414 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:23 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-20T22:30:23.414 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:23 smithi037 ceph-mon[65640]: from='client.? 172.21.15.37:0/2784792619' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:23.414 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:23 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi194.qzjyej-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-20T22:30:23.414 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:23 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:30:23.414 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:23 smithi037 ceph-mon[65640]: from='client.? 172.21.15.37:0/894314240' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T22:30:24.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:24 smithi194 ceph-mon[51725]: Rados config object exists: conf-nfs.foo 2024-09-20T22:30:24.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:24 smithi194 ceph-mon[51725]: Creating key for client.nfs.foo.1.0.smithi194.qzjyej-rgw 2024-09-20T22:30:24.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:24 smithi194 ceph-mon[51725]: Bind address in nfs.foo.1.0.smithi194.qzjyej's ganesha conf is defaulting to empty 2024-09-20T22:30:24.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:24 smithi194 ceph-mon[51725]: Deploying daemon nfs.foo.1.0.smithi194.qzjyej on smithi194 2024-09-20T22:30:24.346 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:24 smithi194 ceph-mon[51725]: from='client.34538 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:30:24.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:24 smithi037 ceph-mon[65640]: Rados config object exists: conf-nfs.foo 2024-09-20T22:30:24.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:24 smithi037 ceph-mon[65640]: Creating key for client.nfs.foo.1.0.smithi194.qzjyej-rgw 2024-09-20T22:30:24.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:24 smithi037 ceph-mon[65640]: Bind address in nfs.foo.1.0.smithi194.qzjyej's ganesha conf is defaulting to empty 2024-09-20T22:30:24.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:24 smithi037 ceph-mon[65640]: Deploying daemon nfs.foo.1.0.smithi194.qzjyej on smithi194 2024-09-20T22:30:24.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:24 smithi037 ceph-mon[65640]: from='client.34538 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:30:25.351 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:25 smithi194 ceph-mon[51725]: pgmap v332: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.6 KiB/s rd, 1.4 KiB/s wr, 4 op/s 2024-09-20T22:30:25.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:25 smithi037 ceph-mon[65640]: pgmap v332: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.6 KiB/s rd, 1.4 KiB/s wr, 4 op/s 2024-09-20T22:30:27.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:26 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:27.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:26 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:27.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:26 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:30:27.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:26 smithi194 ceph-mon[51725]: pgmap v333: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.6 KiB/s rd, 1.4 KiB/s wr, 4 op/s 2024-09-20T22:30:27.141 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:26 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:27.141 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:26 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:27.141 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:26 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:30:27.141 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:26 smithi037 ceph-mon[65640]: pgmap v333: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.6 KiB/s rd, 1.4 KiB/s wr, 4 op/s 2024-09-20T22:30:29.336 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:29 smithi037 ceph-mon[65640]: pgmap v334: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.1 KiB/s rd, 2.0 KiB/s wr, 7 op/s 2024-09-20T22:30:29.336 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:29 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:29.336 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:29 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:29.336 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:29 smithi037 ceph-mon[65640]: from='mds.? [v2:172.21.15.37:6834/1667079263,v1:172.21.15.37:6835/1667079263]' entity='mds.foofs.smithi037.sdtedi' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.194:0/2491128762"}]: dispatch 2024-09-20T22:30:29.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:29 smithi194 ceph-mon[51725]: pgmap v334: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.1 KiB/s rd, 2.0 KiB/s wr, 7 op/s 2024-09-20T22:30:29.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:29 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:29.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:29 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:29.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:29 smithi194 ceph-mon[51725]: from='mds.? [v2:172.21.15.37:6834/1667079263,v1:172.21.15.37:6835/1667079263]' entity='mds.foofs.smithi037.sdtedi' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.194:0/2491128762"}]: dispatch 2024-09-20T22:30:30.290 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:30 smithi037 ceph-mon[65640]: Evicting (and blocklisting) client session 24363 (172.21.15.194:0/2491128762) 2024-09-20T22:30:30.290 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:30 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:30.290 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:30 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:30.290 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:30 smithi037 ceph-mon[65640]: from='mds.? [v2:172.21.15.37:6834/1667079263,v1:172.21.15.37:6835/1667079263]' entity='mds.foofs.smithi037.sdtedi' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.194:0/2491128762"}]': finished 2024-09-20T22:30:30.290 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:30 smithi037 ceph-mon[65640]: osdmap e162: 8 total, 8 up, 8 in 2024-09-20T22:30:30.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:30 smithi194 ceph-mon[51725]: Evicting (and blocklisting) client session 24363 (172.21.15.194:0/2491128762) 2024-09-20T22:30:30.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:30 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:30.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:30 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:30.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:30 smithi194 ceph-mon[51725]: from='mds.? [v2:172.21.15.37:6834/1667079263,v1:172.21.15.37:6835/1667079263]' entity='mds.foofs.smithi037.sdtedi' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.194:0/2491128762"}]': finished 2024-09-20T22:30:30.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:30 smithi194 ceph-mon[51725]: osdmap e162: 8 total, 8 up, 8 in 2024-09-20T22:30:31.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:31 smithi194 ceph-mon[51725]: pgmap v336: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 2.2 KiB/s wr, 8 op/s 2024-09-20T22:30:31.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:31 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:31.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:31 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:31.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:31 smithi037 ceph-mon[65640]: pgmap v336: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 2.2 KiB/s wr, 8 op/s 2024-09-20T22:30:31.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:31 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:31.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:31 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:32.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:32 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:32.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:32 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:32.644 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:32 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:32.644 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:32 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:33.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:33 smithi037 ceph-mon[65640]: pgmap v337: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.1 KiB/s rd, 2.0 KiB/s wr, 7 op/s 2024-09-20T22:30:33.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:33 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:33.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:33 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:33.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:33 smithi194 ceph-mon[51725]: pgmap v337: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.1 KiB/s rd, 2.0 KiB/s wr, 7 op/s 2024-09-20T22:30:33.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:33 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:33.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:33 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:35.534 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:35 smithi037 ceph-mon[65640]: pgmap v338: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.0 KiB/s rd, 1.2 KiB/s wr, 4 op/s 2024-09-20T22:30:35.534 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:35 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:35.534 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:35 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:35.534 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:35 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:30:35.534 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:35 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:30:35.534 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:35 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:35.534 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:35 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:35.534 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:35 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:30:35.535 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:35 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:35.535 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:35 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:35.535 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:35 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:35.535 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:35 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:35.535 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:35 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:35.535 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:35 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:35.535 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:35 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:35.535 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:35 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:35.535 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:35 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:35.535 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:35 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:35.535 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:35 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:35.535 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:35 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:35.535 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:35 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi037.spabdc"}]: dispatch 2024-09-20T22:30:35.535 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:35 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi037.spabdc"}]': finished 2024-09-20T22:30:35.535 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:35 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi194.qzjyej"}]: dispatch 2024-09-20T22:30:35.535 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:35 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi194.qzjyej"}]': finished 2024-09-20T22:30:35.536 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:35 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:35.536 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:35 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:35.536 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:35 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:35.536 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:35 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:35.536 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:35 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:35.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:35 smithi194 ceph-mon[51725]: pgmap v338: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.0 KiB/s rd, 1.2 KiB/s wr, 4 op/s 2024-09-20T22:30:35.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:35 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:35.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:35 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:35.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:35 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:30:35.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:35 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:30:35.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:35 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:35.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:35 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:35.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:35 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:30:35.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:35 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:35.596 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:35 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:35.596 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:35 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:35.596 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:35 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:35.596 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:35 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:35.596 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:35 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:35.596 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:35 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:35.596 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:35 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:35.596 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:35 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:35.596 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:35 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:35.596 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:35 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:35.596 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:35 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:35.596 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:35 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi037.spabdc"}]: dispatch 2024-09-20T22:30:35.596 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:35 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi037.spabdc"}]': finished 2024-09-20T22:30:35.596 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:35 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi194.qzjyej"}]: dispatch 2024-09-20T22:30:35.596 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:35 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi194.qzjyej"}]': finished 2024-09-20T22:30:35.596 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:35 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:35.597 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:35 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:35.597 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:35 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:35.597 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:35 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:35.597 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:35 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:36.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:36 smithi194 ceph-mon[51725]: Upgrade: Setting container_image for all nfs 2024-09-20T22:30:36.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:36 smithi194 ceph-mon[51725]: Upgrade: Setting container_image for all nvmeof 2024-09-20T22:30:36.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:36 smithi037 ceph-mon[65640]: Upgrade: Setting container_image for all nfs 2024-09-20T22:30:36.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:36 smithi037 ceph-mon[65640]: Upgrade: Setting container_image for all nvmeof 2024-09-20T22:30:37.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:37 smithi194 ceph-mon[51725]: pgmap v339: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.0 KiB/s rd, 1.2 KiB/s wr, 4 op/s 2024-09-20T22:30:37.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:37 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:30:37.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:37 smithi037 ceph-mon[65640]: pgmap v339: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.0 KiB/s rd, 1.2 KiB/s wr, 4 op/s 2024-09-20T22:30:37.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:37 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:30:38.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:38 smithi194 ceph-mon[51725]: Upgrade: Updating grafana.smithi037 2024-09-20T22:30:38.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:38 smithi194 ceph-mon[51725]: Deploying daemon grafana.smithi037 on smithi037 2024-09-20T22:30:38.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:38 smithi037 ceph-mon[65640]: Upgrade: Updating grafana.smithi037 2024-09-20T22:30:38.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:38 smithi037 ceph-mon[65640]: Deploying daemon grafana.smithi037 on smithi037 2024-09-20T22:30:39.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:39 smithi194 ceph-mon[51725]: pgmap v340: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 102 B/s rd, 409 B/s wr, 0 op/s 2024-09-20T22:30:39.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:39 smithi037 ceph-mon[65640]: pgmap v340: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 102 B/s rd, 409 B/s wr, 0 op/s 2024-09-20T22:30:41.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:41 smithi194 ceph-mon[51725]: pgmap v341: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 97 B/s rd, 390 B/s wr, 0 op/s 2024-09-20T22:30:41.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:41 smithi037 ceph-mon[65640]: pgmap v341: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 97 B/s rd, 390 B/s wr, 0 op/s 2024-09-20T22:30:43.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:43 smithi194 ceph-mon[51725]: pgmap v342: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 85 B/s rd, 341 B/s wr, 0 op/s 2024-09-20T22:30:43.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:43 smithi037 ceph-mon[65640]: pgmap v342: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 85 B/s rd, 341 B/s wr, 0 op/s 2024-09-20T22:30:45.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:45 smithi037 ceph-mon[65640]: pgmap v343: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 341 B/s wr, 0 op/s 2024-09-20T22:30:45.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:45 smithi194 ceph-mon[51725]: pgmap v343: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 341 B/s wr, 0 op/s 2024-09-20T22:30:47.529 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:47 smithi037 ceph-mon[65640]: pgmap v344: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:30:47.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:47 smithi194 ceph-mon[51725]: pgmap v344: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:30:49.511 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:49 smithi037 ceph-mon[65640]: pgmap v345: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:30:49.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:49 smithi194 ceph-mon[51725]: pgmap v345: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:30:50.844 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:50 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:50.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:50 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:50.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:50 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:30:50.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:50 smithi194 ceph-mon[51725]: pgmap v346: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:30:50.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:50 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:50.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:50 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:50.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:50 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:30:50.913 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:50 smithi037 ceph-mon[65640]: pgmap v346: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:30:52.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:51 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:30:52.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:51 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:30:53.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:52 smithi037 ceph-mon[65640]: pgmap v347: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:30:53.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:52 smithi194 ceph-mon[51725]: pgmap v347: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:30:53.468 INFO:teuthology.orchestra.run.smithi037.stdout:true 2024-09-20T22:30:54.192 INFO:teuthology.orchestra.run.smithi037.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T22:30:54.192 INFO:teuthology.orchestra.run.smithi037.stdout:alertmanager.smithi037 smithi037 *:9093,9094 running (10m) 23s ago 19m 25.5M - 0.25.0 c8568f914cd2 d1f99b3cd707 2024-09-20T22:30:54.192 INFO:teuthology.orchestra.run.smithi037.stdout:crash.smithi037 smithi037 running (7m) 23s ago 19m 6895k - 19.1.1-367-g9e2ac70b cc9251b744a3 77e510d8bdb1 2024-09-20T22:30:54.193 INFO:teuthology.orchestra.run.smithi037.stdout:crash.smithi194 smithi194 running (7m) 25s ago 19m 6895k - 19.1.1-367-g9e2ac70b cc9251b744a3 4d169d1309bc 2024-09-20T22:30:54.193 INFO:teuthology.orchestra.run.smithi037.stdout:grafana.smithi037 smithi037 *:3000 starting - - - - 2024-09-20T22:30:54.193 INFO:teuthology.orchestra.run.smithi037.stdout:mds.foofs.smithi037.sdtedi smithi037 running (69s) 23s ago 13m 30.0M - 19.1.1-367-g9e2ac70b cc9251b744a3 85193a675951 2024-09-20T22:30:54.193 INFO:teuthology.orchestra.run.smithi037.stdout:mds.foofs.smithi194.lkroso smithi194 running (48s) 25s ago 13m 11.9M - 19.1.1-367-g9e2ac70b cc9251b744a3 4e77a613e3bc 2024-09-20T22:30:54.193 INFO:teuthology.orchestra.run.smithi037.stdout:mgr.smithi037.lykblh smithi037 *:8443,9283,8765 running (9m) 23s ago 20m 511M - 19.1.1-367-g9e2ac70b cc9251b744a3 181e46fd9ba6 2024-09-20T22:30:54.193 INFO:teuthology.orchestra.run.smithi037.stdout:mgr.smithi194.bkcglt smithi194 *:8443,9283,8765 running (8m) 25s ago 19m 444M - 19.1.1-367-g9e2ac70b cc9251b744a3 1fa610c97517 2024-09-20T22:30:54.193 INFO:teuthology.orchestra.run.smithi037.stdout:mon.smithi037 smithi037 running (8m) 23s ago 21m 63.0M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 c0229eceb527 2024-09-20T22:30:54.193 INFO:teuthology.orchestra.run.smithi037.stdout:mon.smithi194 smithi194 running (8m) 25s ago 18m 50.8M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 97854807ba91 2024-09-20T22:30:54.193 INFO:teuthology.orchestra.run.smithi037.stdout:nfs.foo.0.0.smithi037.spabdc smithi037 *:2049 running (33s) 23s ago 13m 59.9M - 5.5 cc9251b744a3 7edb20ebb54f 2024-09-20T22:30:54.193 INFO:teuthology.orchestra.run.smithi037.stdout:nfs.foo.1.0.smithi194.qzjyej smithi194 *:2049 running (28s) 25s ago 13m 17.4M - 5.5 cc9251b744a3 a7dc6382dbb5 2024-09-20T22:30:54.193 INFO:teuthology.orchestra.run.smithi037.stdout:node-exporter.smithi037 smithi037 *:9100 running (10m) 23s ago 19m 15.7M - 1.5.0 0da6a335fe13 433e6aaf6137 2024-09-20T22:30:54.193 INFO:teuthology.orchestra.run.smithi037.stdout:node-exporter.smithi194 smithi194 *:9100 running (10m) 25s ago 18m 15.7M - 1.5.0 0da6a335fe13 1d30bb810ad1 2024-09-20T22:30:54.193 INFO:teuthology.orchestra.run.smithi037.stdout:osd.0 smithi037 running (6m) 23s ago 18m 405M 1937M 19.1.1-367-g9e2ac70b cc9251b744a3 c5568bd43c07 2024-09-20T22:30:54.193 INFO:teuthology.orchestra.run.smithi037.stdout:osd.1 smithi037 running (5m) 23s ago 17m 330M 1937M 19.1.1-367-g9e2ac70b cc9251b744a3 f6b715391469 2024-09-20T22:30:54.193 INFO:teuthology.orchestra.run.smithi037.stdout:osd.2 smithi037 running (4m) 23s ago 17m 299M 1937M 19.1.1-367-g9e2ac70b cc9251b744a3 0922768db2a2 2024-09-20T22:30:54.193 INFO:teuthology.orchestra.run.smithi037.stdout:osd.3 smithi037 running (4m) 23s ago 16m 403M 1937M 19.1.1-367-g9e2ac70b cc9251b744a3 312635406c2a 2024-09-20T22:30:54.194 INFO:teuthology.orchestra.run.smithi037.stdout:osd.4 smithi194 running (3m) 25s ago 16m 428M 2704M 19.1.1-367-g9e2ac70b cc9251b744a3 641138d63e50 2024-09-20T22:30:54.194 INFO:teuthology.orchestra.run.smithi037.stdout:osd.5 smithi194 running (2m) 25s ago 15m 335M 2704M 19.1.1-367-g9e2ac70b cc9251b744a3 9a2a44a8db21 2024-09-20T22:30:54.194 INFO:teuthology.orchestra.run.smithi037.stdout:osd.6 smithi194 running (2m) 25s ago 15m 294M 2704M 19.1.1-367-g9e2ac70b cc9251b744a3 2daa9faae9a2 2024-09-20T22:30:54.194 INFO:teuthology.orchestra.run.smithi037.stdout:osd.7 smithi194 running (97s) 25s ago 15m 279M 2704M 19.1.1-367-g9e2ac70b cc9251b744a3 281a2861e7e6 2024-09-20T22:30:54.194 INFO:teuthology.orchestra.run.smithi037.stdout:prometheus.smithi037 smithi037 *:9095 running (9m) 23s ago 18m 63.2M - 2.43.0 a07b618ecd1d 05b826dde2ff 2024-09-20T22:30:54.613 INFO:teuthology.orchestra.run.smithi037.stdout:{ 2024-09-20T22:30:54.613 INFO:teuthology.orchestra.run.smithi037.stdout: "mon": { 2024-09-20T22:30:54.613 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 2 2024-09-20T22:30:54.613 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:30:54.614 INFO:teuthology.orchestra.run.smithi037.stdout: "mgr": { 2024-09-20T22:30:54.614 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 2 2024-09-20T22:30:54.614 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:30:54.614 INFO:teuthology.orchestra.run.smithi037.stdout: "osd": { 2024-09-20T22:30:54.614 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 8 2024-09-20T22:30:54.614 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:30:54.614 INFO:teuthology.orchestra.run.smithi037.stdout: "mds": { 2024-09-20T22:30:54.614 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 2 2024-09-20T22:30:54.614 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:30:54.614 INFO:teuthology.orchestra.run.smithi037.stdout: "overall": { 2024-09-20T22:30:54.614 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 14 2024-09-20T22:30:54.614 INFO:teuthology.orchestra.run.smithi037.stdout: } 2024-09-20T22:30:54.614 INFO:teuthology.orchestra.run.smithi037.stdout:} 2024-09-20T22:30:54.987 INFO:teuthology.orchestra.run.smithi037.stdout:{ 2024-09-20T22:30:54.987 INFO:teuthology.orchestra.run.smithi037.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5", 2024-09-20T22:30:54.987 INFO:teuthology.orchestra.run.smithi037.stdout: "in_progress": true, 2024-09-20T22:30:54.987 INFO:teuthology.orchestra.run.smithi037.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-20T22:30:54.987 INFO:teuthology.orchestra.run.smithi037.stdout: "services_complete": [ 2024-09-20T22:30:54.987 INFO:teuthology.orchestra.run.smithi037.stdout: "mds", 2024-09-20T22:30:54.988 INFO:teuthology.orchestra.run.smithi037.stdout: "crash", 2024-09-20T22:30:54.988 INFO:teuthology.orchestra.run.smithi037.stdout: "osd", 2024-09-20T22:30:54.988 INFO:teuthology.orchestra.run.smithi037.stdout: "mon", 2024-09-20T22:30:54.988 INFO:teuthology.orchestra.run.smithi037.stdout: "mgr", 2024-09-20T22:30:54.988 INFO:teuthology.orchestra.run.smithi037.stdout: "nfs" 2024-09-20T22:30:54.988 INFO:teuthology.orchestra.run.smithi037.stdout: ], 2024-09-20T22:30:54.988 INFO:teuthology.orchestra.run.smithi037.stdout: "progress": "18/23 daemons upgraded", 2024-09-20T22:30:54.988 INFO:teuthology.orchestra.run.smithi037.stdout: "message": "Currently upgrading grafana daemons", 2024-09-20T22:30:54.988 INFO:teuthology.orchestra.run.smithi037.stdout: "is_paused": false 2024-09-20T22:30:54.988 INFO:teuthology.orchestra.run.smithi037.stdout:} 2024-09-20T22:30:55.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:55 smithi194 ceph-mon[51725]: from='client.34552 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:30:55.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:55 smithi194 ceph-mon[51725]: from='client.34556 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:30:55.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:55 smithi194 ceph-mon[51725]: pgmap v348: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-20T22:30:55.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:55 smithi194 ceph-mon[51725]: from='client.34560 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:30:55.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:55 smithi194 ceph-mon[51725]: from='client.? 172.21.15.37:0/2496154364' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:55.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:55 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:55.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:55 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:55.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:55 smithi037 ceph-mon[65640]: from='client.34552 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:30:55.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:55 smithi037 ceph-mon[65640]: from='client.34556 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:30:55.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:55 smithi037 ceph-mon[65640]: pgmap v348: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-20T22:30:55.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:55 smithi037 ceph-mon[65640]: from='client.34560 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:30:55.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:55 smithi037 ceph-mon[65640]: from='client.? 172.21.15.37:0/2496154364' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:55.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:55 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:55.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:55 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:55.437 INFO:teuthology.orchestra.run.smithi037.stdout:HEALTH_OK 2024-09-20T22:30:56.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:56 smithi194 ceph-mon[51725]: from='client.34568 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:30:56.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:56 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:56.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:56 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:56.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:56 smithi194 ceph-mon[51725]: from='client.? 172.21.15.37:0/3991493867' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T22:30:56.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:56 smithi037 ceph-mon[65640]: from='client.34568 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:30:56.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:56 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:56.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:56 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:56.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:56 smithi037 ceph-mon[65640]: from='client.? 172.21.15.37:0/3991493867' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T22:30:57.060 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:57 smithi037 ceph-mon[65640]: pgmap v349: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:30:57.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:57 smithi194 ceph-mon[51725]: pgmap v349: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:30:59.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:59 smithi037 ceph-mon[65640]: pgmap v350: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:30:59.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:59 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:59.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:59 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:59.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:59 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:30:59.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:59 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:30:59.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:59 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:59.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:59 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:59.164 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:59 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-20T22:30:59.164 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:59 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:30:59.164 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:59 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:59.164 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:59 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:59.164 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:59 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:59.164 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:59 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:59.164 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:59 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:59.164 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:59 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:59.164 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:59 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:59.164 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:59 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:59.164 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:59 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:59.164 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:59 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:59.164 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:59 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:59.164 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:59 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:59.164 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:59 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:59.164 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:59 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:59.165 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:59 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:59.165 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:59 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:59.165 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:59 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:59.165 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:59 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:59.165 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:59 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:59.165 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:59 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-20T22:30:59.165 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:59 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-20T22:30:59.165 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:59 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-20T22:30:59.165 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:59 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-20T22:30:59.165 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:59 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-20T22:30:59.165 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:59 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-20T22:30:59.166 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:59 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-20T22:30:59.166 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:59 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-09-20T22:30:59.166 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:59 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-20T22:30:59.166 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:59 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-20T22:30:59.166 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:59 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-20T22:30:59.166 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:59 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-20T22:30:59.166 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:30:59 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-20T22:30:59.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:59 smithi194 ceph-mon[51725]: pgmap v350: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:30:59.346 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:59 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:59.346 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:59 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:59.346 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:59 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:30:59.346 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:59 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:30:59.346 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:59 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:59.346 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:59 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:59.346 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:59 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-20T22:30:59.346 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:59 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:30:59.346 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:59 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:59.346 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:59 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:59.346 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:59 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:59.347 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:59 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:59.347 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:59 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:59.347 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:59 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:59.347 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:59 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:59.347 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:59 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:59.347 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:59 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:59.347 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:59 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:59.347 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:59 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:59.347 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:59 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:59.347 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:59 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:59.347 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:59 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:59.347 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:59 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:59.347 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:59 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:59.347 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:59 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:59.348 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:59 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:30:59.348 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:59 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:30:59.348 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:59 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-20T22:30:59.348 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:59 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-20T22:30:59.348 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:59 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-20T22:30:59.348 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:59 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-20T22:30:59.348 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:59 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-20T22:30:59.348 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:59 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-20T22:30:59.348 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:59 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-20T22:30:59.348 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:59 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-09-20T22:30:59.348 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:59 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-20T22:30:59.348 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:59 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-20T22:30:59.348 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:59 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-20T22:30:59.348 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:59 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-20T22:30:59.348 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:30:59 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-20T22:31:00.320 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:31:00 smithi194 ceph-mon[51725]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-20T22:31:00.320 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:31:00 smithi194 ceph-mon[51725]: Upgrade: Finalizing container_image settings 2024-09-20T22:31:00.320 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:31:00 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-20T22:31:00.321 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:31:00 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-20T22:31:00.321 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:31:00 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-20T22:31:00.321 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:31:00 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-20T22:31:00.321 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:31:00 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-20T22:31:00.321 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:31:00 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-20T22:31:00.321 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:31:00 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-20T22:31:00.321 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:31:00 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-20T22:31:00.321 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:31:00 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-20T22:31:00.321 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:31:00 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-20T22:31:00.321 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:31:00 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-20T22:31:00.321 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:31:00 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-20T22:31:00.321 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:31:00 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-20T22:31:00.321 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:31:00 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-20T22:31:00.321 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:31:00 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-20T22:31:00.321 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:31:00 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-20T22:31:00.321 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:31:00 smithi194 ceph-mon[51725]: Upgrade: Complete! 2024-09-20T22:31:00.322 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:31:00 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-20T22:31:00.322 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:31:00 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-20T22:31:00.322 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:31:00 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:31:00.322 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:31:00 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:31:00.322 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:31:00 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:31:00.322 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:31:00 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:31:00.322 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:31:00 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:31:00.322 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:31:00 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:31:00.322 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:31:00 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:31:00.322 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:31:00 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:31:00.322 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:31:00 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:31:00.322 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:31:00 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:31:00.322 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:31:00 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:31:00.323 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:31:00 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:31:00.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:31:00 smithi037 ceph-mon[65640]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-20T22:31:00.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:31:00 smithi037 ceph-mon[65640]: Upgrade: Finalizing container_image settings 2024-09-20T22:31:00.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:31:00 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-20T22:31:00.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:31:00 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-20T22:31:00.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:31:00 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-20T22:31:00.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:31:00 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-20T22:31:00.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:31:00 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-20T22:31:00.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:31:00 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-20T22:31:00.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:31:00 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-20T22:31:00.414 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:31:00 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-20T22:31:00.414 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:31:00 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-20T22:31:00.414 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:31:00 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-20T22:31:00.414 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:31:00 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-20T22:31:00.414 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:31:00 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-20T22:31:00.414 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:31:00 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-20T22:31:00.414 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:31:00 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-20T22:31:00.414 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:31:00 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-20T22:31:00.414 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:31:00 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-20T22:31:00.414 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:31:00 smithi037 ceph-mon[65640]: Upgrade: Complete! 2024-09-20T22:31:00.414 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:31:00 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-20T22:31:00.414 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:31:00 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-20T22:31:00.417 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:31:00 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:31:00.417 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:31:00 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:31:00.417 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:31:00 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:31:00.417 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:31:00 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:31:00.417 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:31:00 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:31:00.417 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:31:00 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:31:00.417 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:31:00 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:31:00.417 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:31:00 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:31:00.417 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:31:00 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:31:00.417 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:31:00 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:31:00.417 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:31:00 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:31:00.417 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:31:00 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:31:01.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:31:01 smithi194 ceph-mon[51725]: pgmap v351: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:31:01.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:31:01 smithi037 ceph-mon[65640]: pgmap v351: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:31:03.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:31:03 smithi037 ceph-mon[65640]: pgmap v352: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:31:03.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:31:03 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:31:03.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:31:03 smithi194 ceph-mon[51725]: pgmap v352: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:31:03.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:31:03 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:31:05.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:31:05 smithi194 ceph-mon[51725]: pgmap v353: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-20T22:31:05.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:31:05 smithi037 ceph-mon[65640]: pgmap v353: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-20T22:31:07.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:31:07 smithi194 ceph-mon[51725]: pgmap v354: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:31:07.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:31:07 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:31:07.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:31:07 smithi037 ceph-mon[65640]: pgmap v354: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:31:07.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:31:07 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:31:09.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:31:09 smithi194 ceph-mon[51725]: pgmap v355: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:31:09.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:31:09 smithi037 ceph-mon[65640]: pgmap v355: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:31:11.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:31:11 smithi194 ceph-mon[51725]: pgmap v356: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:31:11.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:31:11 smithi037 ceph-mon[65640]: pgmap v356: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:31:13.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:31:13 smithi037 ceph-mon[65640]: pgmap v357: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:31:13.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:31:13 smithi194 ceph-mon[51725]: pgmap v357: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:31:15.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:31:15 smithi194 ceph-mon[51725]: pgmap v358: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-20T22:31:15.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:31:15 smithi037 ceph-mon[65640]: pgmap v358: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-20T22:31:17.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:31:17 smithi194 ceph-mon[51725]: pgmap v359: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:31:17.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:31:17 smithi037 ceph-mon[65640]: pgmap v359: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:31:19.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:31:19 smithi194 ceph-mon[51725]: pgmap v360: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:31:19.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:31:19 smithi037 ceph-mon[65640]: pgmap v360: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:31:21.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:31:21 smithi194 ceph-mon[51725]: pgmap v361: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:31:21.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:31:21 smithi037 ceph-mon[65640]: pgmap v361: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:31:22.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:31:22 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:31:22.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:31:22 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:31:23.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:31:23 smithi037 ceph-mon[65640]: pgmap v362: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:31:23.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:31:23 smithi194 ceph-mon[51725]: pgmap v362: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:31:25.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:31:25 smithi194 ceph-mon[51725]: pgmap v363: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-20T22:31:25.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:31:25 smithi037 ceph-mon[65640]: pgmap v363: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-20T22:31:26.519 DEBUG:teuthology.orchestra.run.smithi037:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph orch ps' 2024-09-20T22:31:27.135 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:31:27 smithi037 ceph-mon[65640]: from='client.34576 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:31:27.135 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:31:27 smithi037 ceph-mon[65640]: pgmap v364: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:31:27.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:31:27 smithi194 ceph-mon[51725]: from='client.34576 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:31:27.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:31:27 smithi194 ceph-mon[51725]: pgmap v364: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:31:28.667 INFO:teuthology.orchestra.run.smithi037.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T22:31:28.668 INFO:teuthology.orchestra.run.smithi037.stdout:alertmanager.smithi037 smithi037 *:9093,9094 running (11m) 34s ago 20m 25.5M - 0.25.0 c8568f914cd2 d1f99b3cd707 2024-09-20T22:31:28.668 INFO:teuthology.orchestra.run.smithi037.stdout:crash.smithi037 smithi037 running (7m) 34s ago 20m 6895k - 19.1.1-367-g9e2ac70b cc9251b744a3 77e510d8bdb1 2024-09-20T22:31:28.668 INFO:teuthology.orchestra.run.smithi037.stdout:crash.smithi194 smithi194 running (7m) 60s ago 19m 6895k - 19.1.1-367-g9e2ac70b cc9251b744a3 4d169d1309bc 2024-09-20T22:31:28.668 INFO:teuthology.orchestra.run.smithi037.stdout:grafana.smithi037 smithi037 *:3000 running (39s) 34s ago 20m 42.2M - 9.4.12 2bacad6d85d8 f5d6ba3e8292 2024-09-20T22:31:28.668 INFO:teuthology.orchestra.run.smithi037.stdout:mds.foofs.smithi037.sdtedi smithi037 running (103s) 34s ago 14m 30.2M - 19.1.1-367-g9e2ac70b cc9251b744a3 85193a675951 2024-09-20T22:31:28.668 INFO:teuthology.orchestra.run.smithi037.stdout:mds.foofs.smithi194.lkroso smithi194 running (83s) 60s ago 14m 11.9M - 19.1.1-367-g9e2ac70b cc9251b744a3 4e77a613e3bc 2024-09-20T22:31:28.668 INFO:teuthology.orchestra.run.smithi037.stdout:mgr.smithi037.lykblh smithi037 *:8443,9283,8765 running (10m) 34s ago 21m 518M - 19.1.1-367-g9e2ac70b cc9251b744a3 181e46fd9ba6 2024-09-20T22:31:28.668 INFO:teuthology.orchestra.run.smithi037.stdout:mgr.smithi194.bkcglt smithi194 *:8443,9283,8765 running (9m) 60s ago 19m 444M - 19.1.1-367-g9e2ac70b cc9251b744a3 1fa610c97517 2024-09-20T22:31:28.668 INFO:teuthology.orchestra.run.smithi037.stdout:mon.smithi037 smithi037 running (9m) 34s ago 21m 63.9M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 c0229eceb527 2024-09-20T22:31:28.668 INFO:teuthology.orchestra.run.smithi037.stdout:mon.smithi194 smithi194 running (8m) 60s ago 19m 50.8M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 97854807ba91 2024-09-20T22:31:28.668 INFO:teuthology.orchestra.run.smithi037.stdout:nfs.foo.0.0.smithi037.spabdc smithi037 *:2049 running (68s) 34s ago 13m 60.2M - 5.5 cc9251b744a3 7edb20ebb54f 2024-09-20T22:31:28.668 INFO:teuthology.orchestra.run.smithi037.stdout:nfs.foo.1.0.smithi194.qzjyej smithi194 *:2049 running (63s) 60s ago 13m 17.4M - 5.5 cc9251b744a3 a7dc6382dbb5 2024-09-20T22:31:28.669 INFO:teuthology.orchestra.run.smithi037.stdout:node-exporter.smithi037 smithi037 *:9100 running (11m) 34s ago 19m 16.0M - 1.5.0 0da6a335fe13 433e6aaf6137 2024-09-20T22:31:28.669 INFO:teuthology.orchestra.run.smithi037.stdout:node-exporter.smithi194 smithi194 *:9100 running (10m) 60s ago 19m 15.7M - 1.5.0 0da6a335fe13 1d30bb810ad1 2024-09-20T22:31:28.669 INFO:teuthology.orchestra.run.smithi037.stdout:osd.0 smithi037 running (7m) 34s ago 18m 406M 1937M 19.1.1-367-g9e2ac70b cc9251b744a3 c5568bd43c07 2024-09-20T22:31:28.669 INFO:teuthology.orchestra.run.smithi037.stdout:osd.1 smithi037 running (6m) 34s ago 18m 330M 1937M 19.1.1-367-g9e2ac70b cc9251b744a3 f6b715391469 2024-09-20T22:31:28.669 INFO:teuthology.orchestra.run.smithi037.stdout:osd.2 smithi037 running (5m) 34s ago 17m 300M 1937M 19.1.1-367-g9e2ac70b cc9251b744a3 0922768db2a2 2024-09-20T22:31:28.669 INFO:teuthology.orchestra.run.smithi037.stdout:osd.3 smithi037 running (4m) 34s ago 17m 403M 1937M 19.1.1-367-g9e2ac70b cc9251b744a3 312635406c2a 2024-09-20T22:31:28.669 INFO:teuthology.orchestra.run.smithi037.stdout:osd.4 smithi194 running (4m) 60s ago 16m 428M 2704M 19.1.1-367-g9e2ac70b cc9251b744a3 641138d63e50 2024-09-20T22:31:28.669 INFO:teuthology.orchestra.run.smithi037.stdout:osd.5 smithi194 running (3m) 60s ago 16m 335M 2704M 19.1.1-367-g9e2ac70b cc9251b744a3 9a2a44a8db21 2024-09-20T22:31:28.669 INFO:teuthology.orchestra.run.smithi037.stdout:osd.6 smithi194 running (2m) 60s ago 16m 294M 2704M 19.1.1-367-g9e2ac70b cc9251b744a3 2daa9faae9a2 2024-09-20T22:31:28.669 INFO:teuthology.orchestra.run.smithi037.stdout:osd.7 smithi194 running (2m) 60s ago 15m 279M 2704M 19.1.1-367-g9e2ac70b cc9251b744a3 281a2861e7e6 2024-09-20T22:31:28.669 INFO:teuthology.orchestra.run.smithi037.stdout:prometheus.smithi037 smithi037 *:9095 running (9m) 34s ago 19m 63.5M - 2.43.0 a07b618ecd1d 05b826dde2ff 2024-09-20T22:31:29.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:31:29 smithi194 ceph-mon[51725]: pgmap v365: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:31:29.346 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:31:29 smithi037 ceph-mon[65640]: pgmap v365: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:31:29.376 DEBUG:teuthology.orchestra.run.smithi037:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph versions' 2024-09-20T22:31:30.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:31:30 smithi194 ceph-mon[51725]: from='client.34580 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:31:30.378 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:31:30 smithi037 ceph-mon[65640]: from='client.34580 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:31:31.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:31:31 smithi194 ceph-mon[51725]: pgmap v366: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:31:31.412 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:31:31 smithi037 ceph-mon[65640]: pgmap v366: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:31:31.513 INFO:teuthology.orchestra.run.smithi037.stdout:{ 2024-09-20T22:31:31.513 INFO:teuthology.orchestra.run.smithi037.stdout: "mon": { 2024-09-20T22:31:31.513 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 2 2024-09-20T22:31:31.513 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:31:31.514 INFO:teuthology.orchestra.run.smithi037.stdout: "mgr": { 2024-09-20T22:31:31.514 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 2 2024-09-20T22:31:31.514 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:31:31.514 INFO:teuthology.orchestra.run.smithi037.stdout: "osd": { 2024-09-20T22:31:31.514 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 8 2024-09-20T22:31:31.514 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:31:31.514 INFO:teuthology.orchestra.run.smithi037.stdout: "mds": { 2024-09-20T22:31:31.514 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 2 2024-09-20T22:31:31.514 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:31:31.514 INFO:teuthology.orchestra.run.smithi037.stdout: "overall": { 2024-09-20T22:31:31.514 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 14 2024-09-20T22:31:31.514 INFO:teuthology.orchestra.run.smithi037.stdout: } 2024-09-20T22:31:31.514 INFO:teuthology.orchestra.run.smithi037.stdout:} 2024-09-20T22:31:32.163 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:31:32 smithi037 ceph-mon[65640]: from='client.? 172.21.15.37:0/3893163637' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:31:32.341 DEBUG:teuthology.orchestra.run.smithi037:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'echo "wait for servicemap items w/ changing names to refresh"' 2024-09-20T22:31:32.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:31:32 smithi194 ceph-mon[51725]: from='client.? 172.21.15.37:0/3893163637' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:31:33.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:31:33 smithi194 ceph-mon[51725]: pgmap v367: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:31:33.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:31:33 smithi037 ceph-mon[65640]: pgmap v367: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:31:34.105 INFO:teuthology.orchestra.run.smithi037.stdout:wait for servicemap items w/ changing names to refresh 2024-09-20T22:31:34.840 DEBUG:teuthology.orchestra.run.smithi037:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'sleep 60' 2024-09-20T22:31:35.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:31:35 smithi194 ceph-mon[51725]: pgmap v368: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-20T22:31:35.386 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:31:35 smithi037 ceph-mon[65640]: pgmap v368: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-20T22:31:37.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:31:37 smithi194 ceph-mon[51725]: pgmap v369: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:31:37.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:31:37 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:31:37.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:31:37 smithi037 ceph-mon[65640]: pgmap v369: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:31:37.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:31:37 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:31:39.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:31:39 smithi194 ceph-mon[51725]: pgmap v370: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:31:39.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:31:39 smithi037 ceph-mon[65640]: pgmap v370: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:31:41.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:31:41 smithi194 ceph-mon[51725]: pgmap v371: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:31:41.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:31:41 smithi037 ceph-mon[65640]: pgmap v371: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:31:43.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:31:43 smithi194 ceph-mon[51725]: pgmap v372: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:31:43.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:31:43 smithi037 ceph-mon[65640]: pgmap v372: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:31:45.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:31:45 smithi194 ceph-mon[51725]: pgmap v373: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-20T22:31:45.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:31:45 smithi037 ceph-mon[65640]: pgmap v373: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-20T22:31:47.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:31:47 smithi194 ceph-mon[51725]: pgmap v374: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:31:47.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:31:47 smithi037 ceph-mon[65640]: pgmap v374: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:31:49.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:31:49 smithi194 ceph-mon[51725]: pgmap v375: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:31:49.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:31:49 smithi037 ceph-mon[65640]: pgmap v375: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:31:51.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:31:51 smithi194 ceph-mon[51725]: pgmap v376: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:31:51.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:31:51 smithi037 ceph-mon[65640]: pgmap v376: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:31:52.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:31:52 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:31:52.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:31:52 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:31:53.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:31:53 smithi037 ceph-mon[65640]: pgmap v377: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:31:53.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:31:53 smithi194 ceph-mon[51725]: pgmap v377: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:31:55.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:31:55 smithi037 ceph-mon[65640]: pgmap v378: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-20T22:31:55.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:31:55 smithi194 ceph-mon[51725]: pgmap v378: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-20T22:31:57.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:31:57 smithi037 ceph-mon[65640]: pgmap v379: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:31:57.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:31:57 smithi194 ceph-mon[51725]: pgmap v379: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:31:59.397 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:31:59 smithi194 ceph-mon[51725]: pgmap v380: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:31:59.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:31:59 smithi037 ceph-mon[65640]: pgmap v380: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:32:00.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:32:00 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:32:00.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:32:00 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:32:00.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:32:00 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:32:00.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:32:00 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:32:00.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:32:00 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:32:00.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:32:00 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:32:01.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:32:01 smithi037 ceph-mon[65640]: pgmap v381: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:32:01.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:32:01 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:32:01.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:32:01 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:32:01.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:32:01 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:32:01.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:32:01 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:32:01.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:32:01 smithi194 ceph-mon[51725]: pgmap v381: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:32:01.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:32:01 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:32:01.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:32:01 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:32:01.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:32:01 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:32:01.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:32:01 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:32:03.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:32:03 smithi037 ceph-mon[65640]: pgmap v382: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:32:03.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:32:03 smithi194 ceph-mon[51725]: pgmap v382: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:32:05.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:32:05 smithi037 ceph-mon[65640]: pgmap v383: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-20T22:32:05.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:32:05 smithi194 ceph-mon[51725]: pgmap v383: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-20T22:32:07.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:32:07 smithi037 ceph-mon[65640]: pgmap v384: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:32:07.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:32:07 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:32:07.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:32:07 smithi194 ceph-mon[51725]: pgmap v384: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:32:07.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:32:07 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:32:09.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:32:09 smithi037 ceph-mon[65640]: pgmap v385: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:32:09.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:32:09 smithi194 ceph-mon[51725]: pgmap v385: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:32:11.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:32:11 smithi037 ceph-mon[65640]: pgmap v386: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:32:11.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:32:11 smithi194 ceph-mon[51725]: pgmap v386: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:32:13.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:32:13 smithi037 ceph-mon[65640]: pgmap v387: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:32:13.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:32:13 smithi194 ceph-mon[51725]: pgmap v387: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:32:15.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:32:15 smithi037 ceph-mon[65640]: pgmap v388: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-20T22:32:15.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:32:15 smithi194 ceph-mon[51725]: pgmap v388: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-20T22:32:17.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:32:17 smithi037 ceph-mon[65640]: pgmap v389: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:32:17.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:32:17 smithi194 ceph-mon[51725]: pgmap v389: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:32:19.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:32:19 smithi037 ceph-mon[65640]: pgmap v390: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:32:19.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:32:19 smithi194 ceph-mon[51725]: pgmap v390: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:32:21.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:32:21 smithi037 ceph-mon[65640]: pgmap v391: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:32:21.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:32:21 smithi194 ceph-mon[51725]: pgmap v391: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:32:22.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:32:22 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:32:22.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:32:22 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:32:23.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:32:23 smithi037 ceph-mon[65640]: pgmap v392: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-20T22:32:23.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:32:23 smithi194 ceph-mon[51725]: pgmap v392: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-20T22:32:25.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:32:25 smithi037 ceph-mon[65640]: pgmap v393: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-20T22:32:25.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:32:25 smithi194 ceph-mon[51725]: pgmap v393: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-20T22:32:27.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:32:27 smithi037 ceph-mon[65640]: pgmap v394: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:32:27.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:32:27 smithi194 ceph-mon[51725]: pgmap v394: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:32:29.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:32:29 smithi194 ceph-mon[51725]: pgmap v395: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:32:29.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:32:29 smithi037 ceph-mon[65640]: pgmap v395: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:32:31.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:32:31 smithi194 ceph-mon[51725]: pgmap v396: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:32:31.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:32:31 smithi037 ceph-mon[65640]: pgmap v396: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:32:33.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:32:33 smithi194 ceph-mon[51725]: pgmap v397: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-20T22:32:33.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:32:33 smithi037 ceph-mon[65640]: pgmap v397: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-20T22:32:35.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:32:35 smithi194 ceph-mon[51725]: pgmap v398: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:32:35.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:32:35 smithi037 ceph-mon[65640]: pgmap v398: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:32:37.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:32:37 smithi037 ceph-mon[65640]: pgmap v399: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:32:37.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:32:37 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:32:37.460 DEBUG:teuthology.orchestra.run.smithi037:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph orch ps' 2024-09-20T22:32:37.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:32:37 smithi194 ceph-mon[51725]: pgmap v399: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:32:37.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:32:37 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:32:39.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:32:39 smithi037 ceph-mon[65640]: pgmap v400: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:32:39.529 INFO:teuthology.orchestra.run.smithi037.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T22:32:39.530 INFO:teuthology.orchestra.run.smithi037.stdout:alertmanager.smithi037 smithi037 *:9093,9094 running (12m) 104s ago 21m 25.5M - 0.25.0 c8568f914cd2 d1f99b3cd707 2024-09-20T22:32:39.530 INFO:teuthology.orchestra.run.smithi037.stdout:crash.smithi037 smithi037 running (9m) 104s ago 21m 6895k - 19.1.1-367-g9e2ac70b cc9251b744a3 77e510d8bdb1 2024-09-20T22:32:39.530 INFO:teuthology.orchestra.run.smithi037.stdout:crash.smithi194 smithi194 running (9m) 2m ago 20m 6895k - 19.1.1-367-g9e2ac70b cc9251b744a3 4d169d1309bc 2024-09-20T22:32:39.530 INFO:teuthology.orchestra.run.smithi037.stdout:grafana.smithi037 smithi037 *:3000 running (110s) 104s ago 21m 42.2M - 9.4.12 2bacad6d85d8 f5d6ba3e8292 2024-09-20T22:32:39.530 INFO:teuthology.orchestra.run.smithi037.stdout:mds.foofs.smithi037.sdtedi smithi037 running (2m) 104s ago 15m 30.2M - 19.1.1-367-g9e2ac70b cc9251b744a3 85193a675951 2024-09-20T22:32:39.530 INFO:teuthology.orchestra.run.smithi037.stdout:mds.foofs.smithi194.lkroso smithi194 running (2m) 2m ago 15m 11.9M - 19.1.1-367-g9e2ac70b cc9251b744a3 4e77a613e3bc 2024-09-20T22:32:39.530 INFO:teuthology.orchestra.run.smithi037.stdout:mgr.smithi037.lykblh smithi037 *:8443,9283,8765 running (11m) 104s ago 22m 518M - 19.1.1-367-g9e2ac70b cc9251b744a3 181e46fd9ba6 2024-09-20T22:32:39.530 INFO:teuthology.orchestra.run.smithi037.stdout:mgr.smithi194.bkcglt smithi194 *:8443,9283,8765 running (10m) 2m ago 20m 444M - 19.1.1-367-g9e2ac70b cc9251b744a3 1fa610c97517 2024-09-20T22:32:39.530 INFO:teuthology.orchestra.run.smithi037.stdout:mon.smithi037 smithi037 running (10m) 104s ago 22m 63.9M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 c0229eceb527 2024-09-20T22:32:39.530 INFO:teuthology.orchestra.run.smithi037.stdout:mon.smithi194 smithi194 running (10m) 2m ago 20m 50.8M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 97854807ba91 2024-09-20T22:32:39.530 INFO:teuthology.orchestra.run.smithi037.stdout:nfs.foo.0.0.smithi037.spabdc smithi037 *:2049 running (2m) 104s ago 15m 60.2M - 5.5 cc9251b744a3 7edb20ebb54f 2024-09-20T22:32:39.530 INFO:teuthology.orchestra.run.smithi037.stdout:nfs.foo.1.0.smithi194.qzjyej smithi194 *:2049 running (2m) 2m ago 15m 17.4M - 5.5 cc9251b744a3 a7dc6382dbb5 2024-09-20T22:32:39.531 INFO:teuthology.orchestra.run.smithi037.stdout:node-exporter.smithi037 smithi037 *:9100 running (12m) 104s ago 21m 16.0M - 1.5.0 0da6a335fe13 433e6aaf6137 2024-09-20T22:32:39.531 INFO:teuthology.orchestra.run.smithi037.stdout:node-exporter.smithi194 smithi194 *:9100 running (11m) 2m ago 20m 15.7M - 1.5.0 0da6a335fe13 1d30bb810ad1 2024-09-20T22:32:39.531 INFO:teuthology.orchestra.run.smithi037.stdout:osd.0 smithi037 running (8m) 104s ago 19m 406M 1937M 19.1.1-367-g9e2ac70b cc9251b744a3 c5568bd43c07 2024-09-20T22:32:39.531 INFO:teuthology.orchestra.run.smithi037.stdout:osd.1 smithi037 running (7m) 104s ago 19m 330M 1937M 19.1.1-367-g9e2ac70b cc9251b744a3 f6b715391469 2024-09-20T22:32:39.531 INFO:teuthology.orchestra.run.smithi037.stdout:osd.2 smithi037 running (6m) 104s ago 18m 300M 1937M 19.1.1-367-g9e2ac70b cc9251b744a3 0922768db2a2 2024-09-20T22:32:39.531 INFO:teuthology.orchestra.run.smithi037.stdout:osd.3 smithi037 running (5m) 104s ago 18m 403M 1937M 19.1.1-367-g9e2ac70b cc9251b744a3 312635406c2a 2024-09-20T22:32:39.531 INFO:teuthology.orchestra.run.smithi037.stdout:osd.4 smithi194 running (5m) 2m ago 18m 428M 2704M 19.1.1-367-g9e2ac70b cc9251b744a3 641138d63e50 2024-09-20T22:32:39.531 INFO:teuthology.orchestra.run.smithi037.stdout:osd.5 smithi194 running (4m) 2m ago 17m 335M 2704M 19.1.1-367-g9e2ac70b cc9251b744a3 9a2a44a8db21 2024-09-20T22:32:39.531 INFO:teuthology.orchestra.run.smithi037.stdout:osd.6 smithi194 running (3m) 2m ago 17m 294M 2704M 19.1.1-367-g9e2ac70b cc9251b744a3 2daa9faae9a2 2024-09-20T22:32:39.531 INFO:teuthology.orchestra.run.smithi037.stdout:osd.7 smithi194 running (3m) 2m ago 16m 279M 2704M 19.1.1-367-g9e2ac70b cc9251b744a3 281a2861e7e6 2024-09-20T22:32:39.531 INFO:teuthology.orchestra.run.smithi037.stdout:prometheus.smithi037 smithi037 *:9095 running (10m) 104s ago 20m 63.5M - 2.43.0 a07b618ecd1d 05b826dde2ff 2024-09-20T22:32:39.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:32:39 smithi194 ceph-mon[51725]: pgmap v400: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:32:40.137 DEBUG:teuthology.orchestra.run.smithi037:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph orch upgrade status' 2024-09-20T22:32:41.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:32:41 smithi037 ceph-mon[65640]: from='client.34588 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:32:41.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:32:41 smithi037 ceph-mon[65640]: pgmap v401: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:32:41.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:32:41 smithi194 ceph-mon[51725]: from='client.34588 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:32:41.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:32:41 smithi194 ceph-mon[51725]: pgmap v401: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:32:42.212 INFO:teuthology.orchestra.run.smithi037.stdout:{ 2024-09-20T22:32:42.213 INFO:teuthology.orchestra.run.smithi037.stdout: "target_image": null, 2024-09-20T22:32:42.213 INFO:teuthology.orchestra.run.smithi037.stdout: "in_progress": false, 2024-09-20T22:32:42.213 INFO:teuthology.orchestra.run.smithi037.stdout: "which": "", 2024-09-20T22:32:42.213 INFO:teuthology.orchestra.run.smithi037.stdout: "services_complete": [], 2024-09-20T22:32:42.213 INFO:teuthology.orchestra.run.smithi037.stdout: "progress": null, 2024-09-20T22:32:42.213 INFO:teuthology.orchestra.run.smithi037.stdout: "message": "", 2024-09-20T22:32:42.213 INFO:teuthology.orchestra.run.smithi037.stdout: "is_paused": false 2024-09-20T22:32:42.213 INFO:teuthology.orchestra.run.smithi037.stdout:} 2024-09-20T22:32:42.913 DEBUG:teuthology.orchestra.run.smithi037:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph health detail' 2024-09-20T22:32:43.453 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:32:43 smithi037 ceph-mon[65640]: pgmap v402: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-20T22:32:43.453 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:32:43 smithi037 ceph-mon[65640]: from='client.34592 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:32:43.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:32:43 smithi194 ceph-mon[51725]: pgmap v402: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-20T22:32:43.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:32:43 smithi194 ceph-mon[51725]: from='client.34592 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:32:45.064 INFO:teuthology.orchestra.run.smithi037.stdout:HEALTH_OK 2024-09-20T22:32:45.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:32:45 smithi037 ceph-mon[65640]: pgmap v403: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:32:45.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:32:45 smithi037 ceph-mon[65640]: from='client.? 172.21.15.37:0/2179021118' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T22:32:45.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:32:45 smithi194 ceph-mon[51725]: pgmap v403: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:32:45.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:32:45 smithi194 ceph-mon[51725]: from='client.? 172.21.15.37:0/2179021118' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T22:32:45.740 DEBUG:teuthology.orchestra.run.smithi037:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph versions' 2024-09-20T22:32:47.366 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:32:47 smithi037 ceph-mon[65640]: pgmap v404: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:32:47.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:32:47 smithi194 ceph-mon[51725]: pgmap v404: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:32:47.922 INFO:teuthology.orchestra.run.smithi037.stdout:{ 2024-09-20T22:32:47.923 INFO:teuthology.orchestra.run.smithi037.stdout: "mon": { 2024-09-20T22:32:47.923 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 2 2024-09-20T22:32:47.923 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:32:47.923 INFO:teuthology.orchestra.run.smithi037.stdout: "mgr": { 2024-09-20T22:32:47.923 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 2 2024-09-20T22:32:47.923 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:32:47.923 INFO:teuthology.orchestra.run.smithi037.stdout: "osd": { 2024-09-20T22:32:47.923 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 8 2024-09-20T22:32:47.923 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:32:47.923 INFO:teuthology.orchestra.run.smithi037.stdout: "mds": { 2024-09-20T22:32:47.923 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 2 2024-09-20T22:32:47.923 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:32:47.923 INFO:teuthology.orchestra.run.smithi037.stdout: "overall": { 2024-09-20T22:32:47.923 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 14 2024-09-20T22:32:47.924 INFO:teuthology.orchestra.run.smithi037.stdout: } 2024-09-20T22:32:47.924 INFO:teuthology.orchestra.run.smithi037.stdout:} 2024-09-20T22:32:48.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:32:48 smithi194 ceph-mon[51725]: from='client.? 172.21.15.37:0/793628585' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:32:48.606 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:32:48 smithi037 ceph-mon[65640]: from='client.? 172.21.15.37:0/793628585' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:32:48.637 DEBUG:teuthology.orchestra.run.smithi037:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph versions | jq -e '"'"'.overall | length == 1'"'"'' 2024-09-20T22:32:49.251 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:32:49 smithi037 ceph-mon[65640]: pgmap v405: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:32:49.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:32:49 smithi194 ceph-mon[51725]: pgmap v405: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:32:50.892 INFO:teuthology.orchestra.run.smithi037.stdout:true 2024-09-20T22:32:51.529 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:32:51 smithi037 ceph-mon[65640]: pgmap v406: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:32:51.530 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:32:51 smithi037 ceph-mon[65640]: from='client.? 172.21.15.37:0/4106693504' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:32:51.557 DEBUG:teuthology.orchestra.run.smithi037:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph versions | jq -e '"'"'.overall | keys'"'"' | grep $sha1' 2024-09-20T22:32:51.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:32:51 smithi194 ceph-mon[51725]: pgmap v406: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:32:51.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:32:51 smithi194 ceph-mon[51725]: from='client.? 172.21.15.37:0/4106693504' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:32:52.545 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:32:52 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:32:52.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:32:52 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:32:53.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:32:53 smithi037 ceph-mon[65640]: pgmap v407: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-20T22:32:53.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:32:53 smithi194 ceph-mon[51725]: pgmap v407: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-20T22:32:53.706 INFO:teuthology.orchestra.run.smithi037.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)" 2024-09-20T22:32:54.395 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:32:54 smithi037 ceph-mon[65640]: from='client.? 172.21.15.37:0/2053641728' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:32:54.425 INFO:teuthology.task.sequential:In sequential, running task cephadm.wait_for_service... 2024-09-20T22:32:54.425 INFO:tasks.cephadm:Waiting for ceph service nfs.foo to start (timeout 300)... 2024-09-20T22:32:54.426 DEBUG:teuthology.orchestra.run.smithi037:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -- ceph orch ls -f json 2024-09-20T22:32:54.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:32:54 smithi194 ceph-mon[51725]: from='client.? 172.21.15.37:0/2053641728' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T22:32:55.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:32:55 smithi037 ceph-mon[65640]: pgmap v408: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:32:55.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:32:55 smithi194 ceph-mon[51725]: pgmap v408: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:32:56.504 INFO:teuthology.orchestra.run.smithi037.stdout: 2024-09-20T22:32:56.504 INFO:teuthology.orchestra.run.smithi037.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-20T22:10:35.834131Z", "last_refresh": "2024-09-20T22:30:54.644439Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-20T22:10:29.668492Z", "last_refresh": "2024-09-20T22:30:28.459029Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "spec": {"anonymous_access": true, "protocol": "https"}, "status": {"created": "2024-09-20T22:10:32.691543Z", "last_refresh": "2024-09-20T22:30:54.644582Z", "ports": [3000], "running": 1, "size": 1}}, {"placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-20T22:17:05.191626Z", "last_refresh": "2024-09-20T22:30:28.459605Z", "running": 2, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-20T22:10:28.369386Z", "last_refresh": "2024-09-20T22:30:28.459123Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi037:172.21.15.37=smithi037", "smithi194:172.21.15.194=smithi194"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-20T22:11:19.388507Z", "last_refresh": "2024-09-20T22:30:28.459195Z", "running": 2, "size": 2}}, {"events": ["2024-09-20T22:32:00.320120Z service:nfs.foo [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foo", "service_name": "nfs.foo", "service_type": "nfs", "spec": {"port": 2049}, "status": {"created": "2024-09-20T22:17:25.887100Z", "last_refresh": "2024-09-20T22:30:28.459674Z", "ports": [2049], "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-20T22:10:34.364687Z", "last_refresh": "2024-09-20T22:30:28.459264Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "cc9251b744a3faafca7c3095f79bfff81873afe823f31a03a11e015a34abcdbd", "container_image_name": "quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5", "last_refresh": "2024-09-20T22:30:28.459332Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-20T22:10:31.440666Z", "last_refresh": "2024-09-20T22:30:54.644721Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-20T22:32:57.135 INFO:tasks.cephadm:nfs.foo has 2/2 2024-09-20T22:32:57.136 DEBUG:teuthology.parallel:result is None 2024-09-20T22:32:57.136 INFO:teuthology.run_tasks:Running task vip.exec... 2024-09-20T22:32:57.146 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi037.front.sepia.ceph.com 2024-09-20T22:32:57.146 DEBUG:teuthology.orchestra.run.smithi037:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'umount /mnt/foo' 2024-09-20T22:32:57.174 INFO:teuthology.orchestra.run.smithi037.stderr:+ umount /mnt/foo 2024-09-20T22:32:57.205 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-20T22:32:57.214 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi037.front.sepia.ceph.com 2024-09-20T22:32:57.214 DEBUG:teuthology.orchestra.run.smithi037:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -- bash -c 'ceph nfs cluster ls | grep foo' 2024-09-20T22:32:57.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:32:57 smithi037 ceph-mon[65640]: pgmap v409: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:32:57.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:32:57 smithi194 ceph-mon[51725]: pgmap v409: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:32:58.500 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:32:58 smithi037 ceph-mon[65640]: from='client.34612 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-20T22:32:58.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:32:58 smithi194 ceph-mon[51725]: from='client.34612 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-20T22:32:59.267 INFO:teuthology.orchestra.run.smithi037.stdout: "foo" 2024-09-20T22:32:59.496 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:32:59 smithi194 ceph-mon[51725]: pgmap v410: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:32:59.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:32:59 smithi037 ceph-mon[65640]: pgmap v410: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:32:59.884 DEBUG:teuthology.orchestra.run.smithi037:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -- bash -c 'ceph nfs export ls foo --detailed' 2024-09-20T22:33:01.469 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:33:01 smithi037 ceph-mon[65640]: from='client.34616 -' entity='client.admin' cmd=[{"prefix": "nfs cluster ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:33:01.469 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:33:01 smithi037 ceph-mon[65640]: pgmap v411: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:33:01.469 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:33:01 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:33:01.469 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:33:01 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:33:01.469 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:33:01 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:33:01.469 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:33:01 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:33:01.469 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:33:01 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:33:01.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:33:01 smithi194 ceph-mon[51725]: from='client.34616 -' entity='client.admin' cmd=[{"prefix": "nfs cluster ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:33:01.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:33:01 smithi194 ceph-mon[51725]: pgmap v411: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T22:33:01.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:33:01 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T22:33:01.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:33:01 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T22:33:01.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:33:01 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T22:33:01.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:33:01 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:33:01.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:33:01 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' 2024-09-20T22:33:01.968 INFO:teuthology.orchestra.run.smithi037.stdout:[ 2024-09-20T22:33:01.969 INFO:teuthology.orchestra.run.smithi037.stdout: { 2024-09-20T22:33:01.969 INFO:teuthology.orchestra.run.smithi037.stdout: "access_type": "RW", 2024-09-20T22:33:01.969 INFO:teuthology.orchestra.run.smithi037.stdout: "clients": [], 2024-09-20T22:33:01.969 INFO:teuthology.orchestra.run.smithi037.stdout: "cluster_id": "foo", 2024-09-20T22:33:01.969 INFO:teuthology.orchestra.run.smithi037.stdout: "export_id": 1, 2024-09-20T22:33:01.969 INFO:teuthology.orchestra.run.smithi037.stdout: "fsal": { 2024-09-20T22:33:01.969 INFO:teuthology.orchestra.run.smithi037.stdout: "fs_name": "foofs", 2024-09-20T22:33:01.969 INFO:teuthology.orchestra.run.smithi037.stdout: "name": "CEPH", 2024-09-20T22:33:01.969 INFO:teuthology.orchestra.run.smithi037.stdout: "user_id": "nfs.foo.1" 2024-09-20T22:33:01.969 INFO:teuthology.orchestra.run.smithi037.stdout: }, 2024-09-20T22:33:01.969 INFO:teuthology.orchestra.run.smithi037.stdout: "path": "/", 2024-09-20T22:33:01.969 INFO:teuthology.orchestra.run.smithi037.stdout: "protocols": [ 2024-09-20T22:33:01.969 INFO:teuthology.orchestra.run.smithi037.stdout: 4 2024-09-20T22:33:01.969 INFO:teuthology.orchestra.run.smithi037.stdout: ], 2024-09-20T22:33:01.969 INFO:teuthology.orchestra.run.smithi037.stdout: "pseudo": "/fake", 2024-09-20T22:33:01.970 INFO:teuthology.orchestra.run.smithi037.stdout: "security_label": true, 2024-09-20T22:33:01.970 INFO:teuthology.orchestra.run.smithi037.stdout: "squash": "none", 2024-09-20T22:33:01.970 INFO:teuthology.orchestra.run.smithi037.stdout: "transports": [ 2024-09-20T22:33:01.970 INFO:teuthology.orchestra.run.smithi037.stdout: "TCP" 2024-09-20T22:33:01.970 INFO:teuthology.orchestra.run.smithi037.stdout: ] 2024-09-20T22:33:01.970 INFO:teuthology.orchestra.run.smithi037.stdout: } 2024-09-20T22:33:01.970 INFO:teuthology.orchestra.run.smithi037.stdout:] 2024-09-20T22:33:02.701 DEBUG:teuthology.orchestra.run.smithi037:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -- bash -c 'rados -p .nfs --all ls -' 2024-09-20T22:33:03.270 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:33:03 smithi037 ceph-mon[65640]: from='client.34620 -' entity='client.admin' cmd=[{"prefix": "nfs export ls", "cluster_id": "foo", "detailed": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:33:03.270 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:33:03 smithi037 ceph-mon[65640]: pgmap v412: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-20T22:33:03.271 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:33:03 smithi037 ceph-mon[65640]: mgrmap e45: smithi037.lykblh(active, since 10m), standbys: smithi194.bkcglt 2024-09-20T22:33:03.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:33:03 smithi194 ceph-mon[51725]: from='client.34620 -' entity='client.admin' cmd=[{"prefix": "nfs export ls", "cluster_id": "foo", "detailed": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T22:33:03.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:33:03 smithi194 ceph-mon[51725]: pgmap v412: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-20T22:33:03.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:33:03 smithi194 ceph-mon[51725]: mgrmap e45: smithi037.lykblh(active, since 10m), standbys: smithi194.bkcglt 2024-09-20T22:33:04.558 INFO:teuthology.orchestra.run.smithi037.stdout:foo rec-0000000000000004:nfs.foo.1 2024-09-20T22:33:04.559 INFO:teuthology.orchestra.run.smithi037.stdout:foo grace 2024-09-20T22:33:04.559 INFO:teuthology.orchestra.run.smithi037.stdout:foo rec-0000000000000004:nfs.foo.0 2024-09-20T22:33:04.559 INFO:teuthology.orchestra.run.smithi037.stdout:foo conf-nfs.foo 2024-09-20T22:33:04.559 INFO:teuthology.orchestra.run.smithi037.stdout:foo export-1 2024-09-20T22:33:05.175 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-20T22:33:05.185 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi037.front.sepia.ceph.com 2024-09-20T22:33:05.185 DEBUG:teuthology.orchestra.run.smithi037:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f9a6357a-779c-11ef-baf5-efdc52797490 -- bash -c 'set -ex 2024-09-20T22:33:05.185 DEBUG:teuthology.orchestra.run.smithi037:> [[ `ceph config get mgr mgr/cephadm/migration_current` -gt 2 ]] 2024-09-20T22:33:05.185 DEBUG:teuthology.orchestra.run.smithi037:> ' 2024-09-20T22:33:05.413 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:33:05 smithi037 ceph-mon[65640]: pgmap v413: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-20T22:33:05.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:33:05 smithi194 ceph-mon[51725]: pgmap v413: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-20T22:33:07.048 INFO:teuthology.orchestra.run.smithi037.stderr:++ ceph config get mgr mgr/cephadm/migration_current 2024-09-20T22:33:07.278 INFO:teuthology.orchestra.run.smithi037.stderr:+ [[ 7 -gt 2 ]] 2024-09-20T22:33:07.594 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:33:07 smithi194 ceph-mon[51725]: pgmap v414: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-20T22:33:07.595 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:33:07 smithi194 ceph-mon[51725]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:33:07.636 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:33:07 smithi037 ceph-mon[65640]: pgmap v414: 97 pgs: 97 active+clean; 226 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-20T22:33:07.636 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:33:07 smithi037 ceph-mon[65640]: from='mgr.34102 172.21.15.37:0/1533942398' entity='mgr.smithi037.lykblh' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T22:33:07.983 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-09-20T22:33:07.993 INFO:tasks.cephadm:Teardown begin 2024-09-20T22:33:07.993 DEBUG:teuthology.orchestra.run.smithi037:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-20T22:33:08.024 DEBUG:teuthology.orchestra.run.smithi194:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-20T22:33:08.057 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-09-20T22:33:08.057 DEBUG:teuthology.orchestra.run.smithi037:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-09-20T22:33:08.083 DEBUG:teuthology.orchestra.run.smithi194:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-09-20T22:33:08.114 INFO:tasks.cephadm:Stopping all daemons... 2024-09-20T22:33:08.114 INFO:tasks.cephadm.mon.smithi037:Stopping mon.smithi037... 2024-09-20T22:33:08.114 DEBUG:teuthology.orchestra.run.smithi037:> sudo systemctl stop ceph-f9a6357a-779c-11ef-baf5-efdc52797490@mon.smithi037 2024-09-20T22:33:08.239 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:33:08 smithi037 systemd[1]: Stopping Ceph mon.smithi037 for f9a6357a-779c-11ef-baf5-efdc52797490... 2024-09-20T22:33:08.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:33:08 smithi194 ceph-mon[51725]: from='client.? 172.21.15.37:0/2243974187' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/cephadm/migration_current"}]: dispatch 2024-09-20T22:33:08.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:33:08 smithi037 ceph-mon[65640]: from='client.? 172.21.15.37:0/2243974187' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/cephadm/migration_current"}]: dispatch 2024-09-20T22:33:08.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:33:08 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-mon-smithi037[65636]: 2024-09-20T22:33:08.446+0000 7f9805236640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi037 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-20T22:33:08.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:33:08 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-mon-smithi037[65636]: 2024-09-20T22:33:08.446+0000 7f9805236640 -1 mon.smithi037@0(leader) e3 *** Got Signal Terminated *** 2024-09-20T22:33:08.663 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:33:08 smithi037 podman[98969]: 2024-09-20 22:33:08.477695855 +0000 UTC m=+0.152999675 container died c0229eceb527050ba8644ebd6d5faff53bf9e2c05aec6ccdd04a9821ec899cf7 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-mon-smithi037, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , RELEASE=squid-9e2ac70, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True) 2024-09-20T22:33:09.116 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:33:08 smithi037 podman[98969]: 2024-09-20 22:33:08.829654604 +0000 UTC m=+0.504958414 container cleanup c0229eceb527050ba8644ebd6d5faff53bf9e2c05aec6ccdd04a9821ec899cf7 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-mon-smithi037, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=squid-9e2ac70, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, GIT_BRANCH=HEAD) 2024-09-20T22:33:09.116 INFO:journalctl@ceph.mon.smithi037.smithi037.stdout:Sep 20 22:33:08 smithi037 bash[98969]: ceph-f9a6357a-779c-11ef-baf5-efdc52797490-mon-smithi037 2024-09-20T22:33:09.273 DEBUG:teuthology.orchestra.run.smithi037:> sudo pkill -f 'journalctl -f -n 0 -u ceph-f9a6357a-779c-11ef-baf5-efdc52797490@mon.smithi037.service' 2024-09-20T22:33:09.325 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-20T22:33:09.325 INFO:tasks.cephadm.mon.smithi037:Stopped mon.smithi037 2024-09-20T22:33:09.325 INFO:tasks.cephadm.mon.smithi194:Stopping mon.smithi194... 2024-09-20T22:33:09.325 DEBUG:teuthology.orchestra.run.smithi194:> sudo systemctl stop ceph-f9a6357a-779c-11ef-baf5-efdc52797490@mon.smithi194 2024-09-20T22:33:09.630 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:33:09 smithi194 systemd[1]: Stopping Ceph mon.smithi194 for f9a6357a-779c-11ef-baf5-efdc52797490... 2024-09-20T22:33:10.030 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:33:09 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-mon-smithi194[51721]: 2024-09-20T22:33:09.627+0000 7fc9f7171640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi194 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-20T22:33:10.030 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:33:09 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-mon-smithi194[51721]: 2024-09-20T22:33:09.627+0000 7fc9f7171640 -1 mon.smithi194@1(peon) e3 *** Got Signal Terminated *** 2024-09-20T22:33:10.030 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:33:09 smithi194 podman[73605]: 2024-09-20 22:33:09.760440526 +0000 UTC m=+0.232373410 container died 97854807ba91a92fcb11515626eb160ee952ac236147761f68cbd8977b8382cd (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-mon-smithi194, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, RELEASE=squid-9e2ac70, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-20T22:33:10.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:33:10 smithi194 podman[73605]: 2024-09-20 22:33:10.20289244 +0000 UTC m=+0.674825364 container cleanup 97854807ba91a92fcb11515626eb160ee952ac236147761f68cbd8977b8382cd (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-mon-smithi194, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-9e2ac70, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-09-20T22:33:10.345 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:33:10 smithi194 bash[73605]: ceph-f9a6357a-779c-11ef-baf5-efdc52797490-mon-smithi194 2024-09-20T22:33:10.653 DEBUG:teuthology.orchestra.run.smithi194:> sudo pkill -f 'journalctl -f -n 0 -u ceph-f9a6357a-779c-11ef-baf5-efdc52797490@mon.smithi194.service' 2024-09-20T22:33:10.674 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:33:10 smithi194 podman[73618]: 2024-09-20 22:33:10.42123138 +0000 UTC m=+0.658323692 container remove 97854807ba91a92fcb11515626eb160ee952ac236147761f68cbd8977b8382cd (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-mon-smithi194, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=squid-9e2ac70, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True) 2024-09-20T22:33:10.674 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:33:10 smithi194 systemd[1]: ceph-f9a6357a-779c-11ef-baf5-efdc52797490@mon.smithi194.service: Deactivated successfully. 2024-09-20T22:33:10.674 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:33:10 smithi194 systemd[1]: Stopped Ceph mon.smithi194 for f9a6357a-779c-11ef-baf5-efdc52797490. 2024-09-20T22:33:10.674 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 20 22:33:10 smithi194 systemd[1]: ceph-f9a6357a-779c-11ef-baf5-efdc52797490@mon.smithi194.service: Consumed 5.845s CPU time. 2024-09-20T22:33:10.704 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-20T22:33:10.704 INFO:tasks.cephadm.mon.smithi194:Stopped mon.smithi194 2024-09-20T22:33:10.704 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-09-20T22:33:10.704 DEBUG:teuthology.orchestra.run.smithi037:> sudo systemctl stop ceph-f9a6357a-779c-11ef-baf5-efdc52797490@osd.0 2024-09-20T22:33:11.011 INFO:journalctl@ceph.osd.0.smithi037.stdout:Sep 20 22:33:10 smithi037 systemd[1]: Stopping Ceph osd.0 for f9a6357a-779c-11ef-baf5-efdc52797490... 2024-09-20T22:33:11.413 INFO:journalctl@ceph.osd.0.smithi037.stdout:Sep 20 22:33:11 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-0[71518]: 2024-09-20T22:33:11.009+0000 7fb3ba6d1640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-20T22:33:11.413 INFO:journalctl@ceph.osd.0.smithi037.stdout:Sep 20 22:33:11 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-0[71518]: 2024-09-20T22:33:11.009+0000 7fb3ba6d1640 -1 osd.0 162 *** Got signal Terminated *** 2024-09-20T22:33:11.413 INFO:journalctl@ceph.osd.0.smithi037.stdout:Sep 20 22:33:11 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-0[71518]: 2024-09-20T22:33:11.009+0000 7fb3ba6d1640 -1 osd.0 162 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-20T22:33:16.636 INFO:journalctl@ceph.osd.0.smithi037.stdout:Sep 20 22:33:16 smithi037 podman[99103]: 2024-09-20 22:33:16.378733454 +0000 UTC m=+5.467977007 container died c5568bd43c075314ec5f7e5f230f8a40319bd6033ce439b6ff40421150c50a20 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-0, RELEASE=squid-9e2ac70, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=) 2024-09-20T22:33:16.913 INFO:journalctl@ceph.osd.0.smithi037.stdout:Sep 20 22:33:16 smithi037 podman[99103]: 2024-09-20 22:33:16.71735131 +0000 UTC m=+5.806594850 container cleanup c5568bd43c075314ec5f7e5f230f8a40319bd6033ce439b6ff40421150c50a20 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=squid-9e2ac70) 2024-09-20T22:33:16.913 INFO:journalctl@ceph.osd.0.smithi037.stdout:Sep 20 22:33:16 smithi037 bash[99103]: ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-0 2024-09-20T22:33:17.388 INFO:journalctl@ceph.osd.0.smithi037.stdout:Sep 20 22:33:16 smithi037 podman[99153]: 2024-09-20 22:33:16.944172524 +0000 UTC m=+0.561158704 container remove c5568bd43c075314ec5f7e5f230f8a40319bd6033ce439b6ff40421150c50a20 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-0, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=squid-9e2ac70, ceph=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux ) 2024-09-20T22:33:17.663 INFO:journalctl@ceph.osd.0.smithi037.stdout:Sep 20 22:33:17 smithi037 podman[99241]: 2024-09-20 22:33:17.290131598 +0000 UTC m=+0.016593823 image pull cc9251b744a3faafca7c3095f79bfff81873afe823f31a03a11e015a34abcdbd quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5 2024-09-20T22:33:17.663 INFO:journalctl@ceph.osd.0.smithi037.stdout:Sep 20 22:33:17 smithi037 podman[99241]: 2024-09-20 22:33:17.420181099 +0000 UTC m=+0.146643354 container create ff98de5bd50a9c08ecb436f53aaf625bfd675ee20b52ad35e1499d8f363332ea (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-0-deactivate, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, RELEASE=squid-9e2ac70, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-09-20T22:33:18.163 INFO:journalctl@ceph.osd.0.smithi037.stdout:Sep 20 22:33:17 smithi037 podman[99241]: 2024-09-20 22:33:17.69776666 +0000 UTC m=+0.424228882 container init ff98de5bd50a9c08ecb436f53aaf625bfd675ee20b52ad35e1499d8f363332ea (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-0-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-9e2ac70, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, ceph=True) 2024-09-20T22:33:18.163 INFO:journalctl@ceph.osd.0.smithi037.stdout:Sep 20 22:33:17 smithi037 podman[99241]: 2024-09-20 22:33:17.701366945 +0000 UTC m=+0.427829161 container start ff98de5bd50a9c08ecb436f53aaf625bfd675ee20b52ad35e1499d8f363332ea (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-0-deactivate, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, RELEASE=squid-9e2ac70, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-09-20T22:33:18.163 INFO:journalctl@ceph.osd.0.smithi037.stdout:Sep 20 22:33:17 smithi037 podman[99241]: 2024-09-20 22:33:17.789453056 +0000 UTC m=+0.515915273 container attach ff98de5bd50a9c08ecb436f53aaf625bfd675ee20b52ad35e1499d8f363332ea (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-0-deactivate, ceph=True, org.label-schema.schema-version=1.0, RELEASE=squid-9e2ac70, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux ) 2024-09-20T22:33:18.163 INFO:journalctl@ceph.osd.0.smithi037.stdout:Sep 20 22:33:17 smithi037 podman[99241]: 2024-09-20 22:33:17.901622895 +0000 UTC m=+0.628085112 container died ff98de5bd50a9c08ecb436f53aaf625bfd675ee20b52ad35e1499d8f363332ea (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-0-deactivate, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=squid-9e2ac70, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-09-20T22:33:18.586 DEBUG:teuthology.orchestra.run.smithi037:> sudo pkill -f 'journalctl -f -n 0 -u ceph-f9a6357a-779c-11ef-baf5-efdc52797490@osd.0.service' 2024-09-20T22:33:18.628 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-20T22:33:18.628 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-09-20T22:33:18.628 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-09-20T22:33:18.628 DEBUG:teuthology.orchestra.run.smithi037:> sudo systemctl stop ceph-f9a6357a-779c-11ef-baf5-efdc52797490@osd.1 2024-09-20T22:33:18.913 INFO:journalctl@ceph.osd.1.smithi037.stdout:Sep 20 22:33:18 smithi037 systemd[1]: Stopping Ceph osd.1 for f9a6357a-779c-11ef-baf5-efdc52797490... 2024-09-20T22:33:19.413 INFO:journalctl@ceph.osd.1.smithi037.stdout:Sep 20 22:33:18 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-1[74227]: 2024-09-20T22:33:18.979+0000 7faa9e6fc640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-20T22:33:19.413 INFO:journalctl@ceph.osd.1.smithi037.stdout:Sep 20 22:33:18 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-1[74227]: 2024-09-20T22:33:18.979+0000 7faa9e6fc640 -1 osd.1 162 *** Got signal Terminated *** 2024-09-20T22:33:19.413 INFO:journalctl@ceph.osd.1.smithi037.stdout:Sep 20 22:33:18 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-1[74227]: 2024-09-20T22:33:18.979+0000 7faa9e6fc640 -1 osd.1 162 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-20T22:33:24.657 INFO:journalctl@ceph.osd.1.smithi037.stdout:Sep 20 22:33:24 smithi037 podman[99344]: 2024-09-20 22:33:24.362182467 +0000 UTC m=+5.497439009 container died f6b715391469057d4b1bfd72f6271784f6be28201e027a8bc5421437e9d85474 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-1, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-9e2ac70, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-20T22:33:24.913 INFO:journalctl@ceph.osd.1.smithi037.stdout:Sep 20 22:33:24 smithi037 podman[99344]: 2024-09-20 22:33:24.762919811 +0000 UTC m=+5.898176351 container cleanup f6b715391469057d4b1bfd72f6271784f6be28201e027a8bc5421437e9d85474 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-1, RELEASE=squid-9e2ac70, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-20T22:33:24.913 INFO:journalctl@ceph.osd.1.smithi037.stdout:Sep 20 22:33:24 smithi037 bash[99344]: ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-1 2024-09-20T22:33:25.333 INFO:journalctl@ceph.osd.1.smithi037.stdout:Sep 20 22:33:25 smithi037 podman[99359]: 2024-09-20 22:33:25.006077625 +0000 UTC m=+0.639857553 container remove f6b715391469057d4b1bfd72f6271784f6be28201e027a8bc5421437e9d85474 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-1, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=squid-9e2ac70, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True) 2024-09-20T22:33:25.636 INFO:journalctl@ceph.osd.1.smithi037.stdout:Sep 20 22:33:25 smithi037 podman[99451]: 2024-09-20 22:33:25.332000013 +0000 UTC m=+0.099945082 container create 4aaa2f89ac80055807d117fa29c662fb019a75f9995c9e17b4c0c3daa78f3e58 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-1-deactivate, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=squid-9e2ac70, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-09-20T22:33:25.636 INFO:journalctl@ceph.osd.1.smithi037.stdout:Sep 20 22:33:25 smithi037 podman[99451]: 2024-09-20 22:33:25.248664636 +0000 UTC m=+0.016609669 image pull cc9251b744a3faafca7c3095f79bfff81873afe823f31a03a11e015a34abcdbd quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5 2024-09-20T22:33:25.636 INFO:journalctl@ceph.osd.1.smithi037.stdout:Sep 20 22:33:25 smithi037 podman[99451]: 2024-09-20 22:33:25.551584302 +0000 UTC m=+0.319529330 container init 4aaa2f89ac80055807d117fa29c662fb019a75f9995c9e17b4c0c3daa78f3e58 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-1-deactivate, ceph=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, RELEASE=squid-9e2ac70, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-09-20T22:33:25.636 INFO:journalctl@ceph.osd.1.smithi037.stdout:Sep 20 22:33:25 smithi037 podman[99451]: 2024-09-20 22:33:25.555012639 +0000 UTC m=+0.322957663 container start 4aaa2f89ac80055807d117fa29c662fb019a75f9995c9e17b4c0c3daa78f3e58 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-1-deactivate, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, RELEASE=squid-9e2ac70, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-09-20T22:33:25.636 INFO:journalctl@ceph.osd.1.smithi037.stdout:Sep 20 22:33:25 smithi037 podman[99451]: 2024-09-20 22:33:25.634760525 +0000 UTC m=+0.402705558 container attach 4aaa2f89ac80055807d117fa29c662fb019a75f9995c9e17b4c0c3daa78f3e58 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-1-deactivate, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=squid-9e2ac70) 2024-09-20T22:33:25.913 INFO:journalctl@ceph.osd.1.smithi037.stdout:Sep 20 22:33:25 smithi037 podman[99451]: 2024-09-20 22:33:25.750808805 +0000 UTC m=+0.518753828 container died 4aaa2f89ac80055807d117fa29c662fb019a75f9995c9e17b4c0c3daa78f3e58 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-1-deactivate, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, RELEASE=squid-9e2ac70, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-20T22:33:26.362 DEBUG:teuthology.orchestra.run.smithi037:> sudo pkill -f 'journalctl -f -n 0 -u ceph-f9a6357a-779c-11ef-baf5-efdc52797490@osd.1.service' 2024-09-20T22:33:26.406 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-20T22:33:26.407 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-09-20T22:33:26.407 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-09-20T22:33:26.407 DEBUG:teuthology.orchestra.run.smithi037:> sudo systemctl stop ceph-f9a6357a-779c-11ef-baf5-efdc52797490@osd.2 2024-09-20T22:33:26.663 INFO:journalctl@ceph.osd.2.smithi037.stdout:Sep 20 22:33:26 smithi037 systemd[1]: Stopping Ceph osd.2 for f9a6357a-779c-11ef-baf5-efdc52797490... 2024-09-20T22:33:27.163 INFO:journalctl@ceph.osd.2.smithi037.stdout:Sep 20 22:33:26 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-2[76871]: 2024-09-20T22:33:26.791+0000 7f31c1de5640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-20T22:33:27.163 INFO:journalctl@ceph.osd.2.smithi037.stdout:Sep 20 22:33:26 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-2[76871]: 2024-09-20T22:33:26.791+0000 7f31c1de5640 -1 osd.2 162 *** Got signal Terminated *** 2024-09-20T22:33:27.163 INFO:journalctl@ceph.osd.2.smithi037.stdout:Sep 20 22:33:26 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-2[76871]: 2024-09-20T22:33:26.791+0000 7f31c1de5640 -1 osd.2 162 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-20T22:33:32.413 INFO:journalctl@ceph.osd.2.smithi037.stdout:Sep 20 22:33:32 smithi037 podman[99553]: 2024-09-20 22:33:32.214371346 +0000 UTC m=+5.577867449 container died 0922768db2a25e2e82f6de041478e2da194761b05e96027d4eed853dfaf56c76 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-2, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=squid-9e2ac70, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_CLEAN=True) 2024-09-20T22:33:32.827 INFO:journalctl@ceph.osd.2.smithi037.stdout:Sep 20 22:33:32 smithi037 podman[99553]: 2024-09-20 22:33:32.616705408 +0000 UTC m=+5.980201500 container cleanup 0922768db2a25e2e82f6de041478e2da194761b05e96027d4eed853dfaf56c76 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-2, RELEASE=squid-9e2ac70, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-20T22:33:32.827 INFO:journalctl@ceph.osd.2.smithi037.stdout:Sep 20 22:33:32 smithi037 bash[99553]: ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-2 2024-09-20T22:33:33.163 INFO:journalctl@ceph.osd.2.smithi037.stdout:Sep 20 22:33:32 smithi037 podman[99571]: 2024-09-20 22:33:32.868196404 +0000 UTC m=+0.647524858 container remove 0922768db2a25e2e82f6de041478e2da194761b05e96027d4eed853dfaf56c76 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=squid-9e2ac70) 2024-09-20T22:33:33.507 INFO:journalctl@ceph.osd.2.smithi037.stdout:Sep 20 22:33:33 smithi037 podman[99662]: 2024-09-20 22:33:33.151020779 +0000 UTC m=+0.017510631 image pull cc9251b744a3faafca7c3095f79bfff81873afe823f31a03a11e015a34abcdbd quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5 2024-09-20T22:33:33.507 INFO:journalctl@ceph.osd.2.smithi037.stdout:Sep 20 22:33:33 smithi037 podman[99662]: 2024-09-20 22:33:33.285875866 +0000 UTC m=+0.152365749 container create e279534a3d999c40c2648056956f5103b2cee9d9c97c1f5d34821f9dc5e80f74 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-2-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, RELEASE=squid-9e2ac70, GIT_BRANCH=HEAD) 2024-09-20T22:33:33.507 INFO:journalctl@ceph.osd.2.smithi037.stdout:Sep 20 22:33:33 smithi037 podman[99662]: 2024-09-20 22:33:33.506096291 +0000 UTC m=+0.372586140 container init e279534a3d999c40c2648056956f5103b2cee9d9c97c1f5d34821f9dc5e80f74 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-2-deactivate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, ceph=True, RELEASE=squid-9e2ac70, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.license=GPLv2) 2024-09-20T22:33:33.913 INFO:journalctl@ceph.osd.2.smithi037.stdout:Sep 20 22:33:33 smithi037 podman[99662]: 2024-09-20 22:33:33.510110756 +0000 UTC m=+0.376600605 container start e279534a3d999c40c2648056956f5103b2cee9d9c97c1f5d34821f9dc5e80f74 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-2-deactivate, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, RELEASE=squid-9e2ac70, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=) 2024-09-20T22:33:33.913 INFO:journalctl@ceph.osd.2.smithi037.stdout:Sep 20 22:33:33 smithi037 podman[99662]: 2024-09-20 22:33:33.604940836 +0000 UTC m=+0.471430686 container attach e279534a3d999c40c2648056956f5103b2cee9d9c97c1f5d34821f9dc5e80f74 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-2-deactivate, ceph=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=squid-9e2ac70, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-20T22:33:33.913 INFO:journalctl@ceph.osd.2.smithi037.stdout:Sep 20 22:33:33 smithi037 podman[99662]: 2024-09-20 22:33:33.701233574 +0000 UTC m=+0.567723427 container died e279534a3d999c40c2648056956f5103b2cee9d9c97c1f5d34821f9dc5e80f74 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-2-deactivate, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, ceph=True, RELEASE=squid-9e2ac70, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-09-20T22:33:34.354 DEBUG:teuthology.orchestra.run.smithi037:> sudo pkill -f 'journalctl -f -n 0 -u ceph-f9a6357a-779c-11ef-baf5-efdc52797490@osd.2.service' 2024-09-20T22:33:34.398 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-20T22:33:34.398 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-09-20T22:33:34.398 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-09-20T22:33:34.398 DEBUG:teuthology.orchestra.run.smithi037:> sudo systemctl stop ceph-f9a6357a-779c-11ef-baf5-efdc52797490@osd.3 2024-09-20T22:33:34.663 INFO:journalctl@ceph.osd.3.smithi037.stdout:Sep 20 22:33:34 smithi037 systemd[1]: Stopping Ceph osd.3 for f9a6357a-779c-11ef-baf5-efdc52797490... 2024-09-20T22:33:35.163 INFO:journalctl@ceph.osd.3.smithi037.stdout:Sep 20 22:33:34 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-3[79589]: 2024-09-20T22:33:34.745+0000 7f0b7a004640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-20T22:33:35.163 INFO:journalctl@ceph.osd.3.smithi037.stdout:Sep 20 22:33:34 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-3[79589]: 2024-09-20T22:33:34.745+0000 7f0b7a004640 -1 osd.3 162 *** Got signal Terminated *** 2024-09-20T22:33:35.163 INFO:journalctl@ceph.osd.3.smithi037.stdout:Sep 20 22:33:34 smithi037 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-3[79589]: 2024-09-20T22:33:34.745+0000 7f0b7a004640 -1 osd.3 162 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-20T22:33:37.095 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 20 22:33:36 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-4[56643]: 2024-09-20T22:33:36.681+0000 7f8b05516640 -1 osd.4 162 heartbeat_check: no reply from 172.21.15.37:6806 osd.0 since back 2024-09-20T22:33:11.385940+0000 front 2024-09-20T22:33:11.385925+0000 (oldest deadline 2024-09-20T22:33:36.586139+0000) 2024-09-20T22:33:37.844 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 20 22:33:37 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-4[56643]: 2024-09-20T22:33:37.718+0000 7f8b05516640 -1 osd.4 162 heartbeat_check: no reply from 172.21.15.37:6806 osd.0 since back 2024-09-20T22:33:11.385940+0000 front 2024-09-20T22:33:11.385925+0000 (oldest deadline 2024-09-20T22:33:36.586139+0000) 2024-09-20T22:33:38.595 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:33:38 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6[61138]: 2024-09-20T22:33:38.339+0000 7f595101c640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.37:6806 osd.0 since back 2024-09-20T22:33:14.174345+0000 front 2024-09-20T22:33:14.174182+0000 (oldest deadline 2024-09-20T22:33:38.274185+0000) 2024-09-20T22:33:39.094 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 20 22:33:38 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-4[56643]: 2024-09-20T22:33:38.723+0000 7f8b05516640 -1 osd.4 162 heartbeat_check: no reply from 172.21.15.37:6806 osd.0 since back 2024-09-20T22:33:11.385940+0000 front 2024-09-20T22:33:11.385925+0000 (oldest deadline 2024-09-20T22:33:36.586139+0000) 2024-09-20T22:33:39.595 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:33:39 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6[61138]: 2024-09-20T22:33:39.307+0000 7f595101c640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.37:6806 osd.0 since back 2024-09-20T22:33:14.174345+0000 front 2024-09-20T22:33:14.174182+0000 (oldest deadline 2024-09-20T22:33:38.274185+0000) 2024-09-20T22:33:40.094 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 20 22:33:39 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-4[56643]: 2024-09-20T22:33:39.716+0000 7f8b05516640 -1 osd.4 162 heartbeat_check: no reply from 172.21.15.37:6806 osd.0 since back 2024-09-20T22:33:11.385940+0000 front 2024-09-20T22:33:11.385925+0000 (oldest deadline 2024-09-20T22:33:36.586139+0000) 2024-09-20T22:33:40.595 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:33:40 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-5[58907]: 2024-09-20T22:33:40.475+0000 7f63e8804640 -1 osd.5 162 heartbeat_check: no reply from 172.21.15.37:6806 osd.0 since back 2024-09-20T22:33:14.098648+0000 front 2024-09-20T22:33:14.098917+0000 (oldest deadline 2024-09-20T22:33:39.998833+0000) 2024-09-20T22:33:40.595 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:33:40 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:33:40.232+0000 7f262c93f640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.37:6806 osd.0 since back 2024-09-20T22:33:15.226504+0000 front 2024-09-20T22:33:15.226384+0000 (oldest deadline 2024-09-20T22:33:39.326239+0000) 2024-09-20T22:33:40.595 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:33:40 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6[61138]: 2024-09-20T22:33:40.342+0000 7f595101c640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.37:6806 osd.0 since back 2024-09-20T22:33:14.174345+0000 front 2024-09-20T22:33:14.174182+0000 (oldest deadline 2024-09-20T22:33:38.274185+0000) 2024-09-20T22:33:40.648 INFO:journalctl@ceph.osd.3.smithi037.stdout:Sep 20 22:33:40 smithi037 podman[99763]: 2024-09-20 22:33:40.32386411 +0000 UTC m=+5.696276339 container died 312635406c2a164621eb12033e3b5b43056fc63bb70cf3463419d85f9b735709 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-3, ceph=True, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=squid-9e2ac70, org.label-schema.build-date=20240731) 2024-09-20T22:33:40.913 INFO:journalctl@ceph.osd.3.smithi037.stdout:Sep 20 22:33:40 smithi037 podman[99763]: 2024-09-20 22:33:40.736388466 +0000 UTC m=+6.108800687 container cleanup 312635406c2a164621eb12033e3b5b43056fc63bb70cf3463419d85f9b735709 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-3, ceph=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=squid-9e2ac70, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-20T22:33:40.913 INFO:journalctl@ceph.osd.3.smithi037.stdout:Sep 20 22:33:40 smithi037 bash[99763]: ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-3 2024-09-20T22:33:41.095 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 20 22:33:40 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-4[56643]: 2024-09-20T22:33:40.760+0000 7f8b05516640 -1 osd.4 162 heartbeat_check: no reply from 172.21.15.37:6806 osd.0 since back 2024-09-20T22:33:11.385940+0000 front 2024-09-20T22:33:11.385925+0000 (oldest deadline 2024-09-20T22:33:36.586139+0000) 2024-09-20T22:33:41.371 INFO:journalctl@ceph.osd.3.smithi037.stdout:Sep 20 22:33:40 smithi037 podman[99810]: 2024-09-20 22:33:40.946079578 +0000 UTC m=+0.623927260 container remove 312635406c2a164621eb12033e3b5b43056fc63bb70cf3463419d85f9b735709 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-3, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=squid-9e2ac70, ceph=True) 2024-09-20T22:33:41.527 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:33:41 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6[61138]: 2024-09-20T22:33:41.355+0000 7f595101c640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.37:6806 osd.0 since back 2024-09-20T22:33:14.174345+0000 front 2024-09-20T22:33:14.174182+0000 (oldest deadline 2024-09-20T22:33:38.274185+0000) 2024-09-20T22:33:41.528 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:33:41 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:33:41.193+0000 7f262c93f640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.37:6806 osd.0 since back 2024-09-20T22:33:15.226504+0000 front 2024-09-20T22:33:15.226384+0000 (oldest deadline 2024-09-20T22:33:39.326239+0000) 2024-09-20T22:33:41.663 INFO:journalctl@ceph.osd.3.smithi037.stdout:Sep 20 22:33:41 smithi037 podman[99899]: 2024-09-20 22:33:41.272850417 +0000 UTC m=+0.016917933 image pull cc9251b744a3faafca7c3095f79bfff81873afe823f31a03a11e015a34abcdbd quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5 2024-09-20T22:33:41.663 INFO:journalctl@ceph.osd.3.smithi037.stdout:Sep 20 22:33:41 smithi037 podman[99899]: 2024-09-20 22:33:41.396906698 +0000 UTC m=+0.140974197 container create 9d008a0a9eb30f90030fe478e9add229613617fdf0ef6b62f571d0c934000308 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-3-deactivate, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, RELEASE=squid-9e2ac70, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-09-20T22:33:41.663 INFO:journalctl@ceph.osd.3.smithi037.stdout:Sep 20 22:33:41 smithi037 podman[99899]: 2024-09-20 22:33:41.599435456 +0000 UTC m=+0.343502965 container init 9d008a0a9eb30f90030fe478e9add229613617fdf0ef6b62f571d0c934000308 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-3-deactivate, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, RELEASE=squid-9e2ac70, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-20T22:33:41.663 INFO:journalctl@ceph.osd.3.smithi037.stdout:Sep 20 22:33:41 smithi037 podman[99899]: 2024-09-20 22:33:41.602807049 +0000 UTC m=+0.346874551 container start 9d008a0a9eb30f90030fe478e9add229613617fdf0ef6b62f571d0c934000308 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-3-deactivate, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, RELEASE=squid-9e2ac70, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-20T22:33:41.807 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:33:41 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-5[58907]: 2024-09-20T22:33:41.525+0000 7f63e8804640 -1 osd.5 162 heartbeat_check: no reply from 172.21.15.37:6806 osd.0 since back 2024-09-20T22:33:14.098648+0000 front 2024-09-20T22:33:14.098917+0000 (oldest deadline 2024-09-20T22:33:39.998833+0000) 2024-09-20T22:33:42.094 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 20 22:33:41 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-4[56643]: 2024-09-20T22:33:41.806+0000 7f8b05516640 -1 osd.4 162 heartbeat_check: no reply from 172.21.15.37:6806 osd.0 since back 2024-09-20T22:33:11.385940+0000 front 2024-09-20T22:33:11.385925+0000 (oldest deadline 2024-09-20T22:33:36.586139+0000) 2024-09-20T22:33:42.122 INFO:journalctl@ceph.osd.3.smithi037.stdout:Sep 20 22:33:41 smithi037 podman[99899]: 2024-09-20 22:33:41.691898418 +0000 UTC m=+0.435965932 container attach 9d008a0a9eb30f90030fe478e9add229613617fdf0ef6b62f571d0c934000308 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-3-deactivate, GIT_BRANCH=HEAD, RELEASE=squid-9e2ac70, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True) 2024-09-20T22:33:42.122 INFO:journalctl@ceph.osd.3.smithi037.stdout:Sep 20 22:33:41 smithi037 podman[99899]: 2024-09-20 22:33:41.799675915 +0000 UTC m=+0.543743415 container died 9d008a0a9eb30f90030fe478e9add229613617fdf0ef6b62f571d0c934000308 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-3-deactivate, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, RELEASE=squid-9e2ac70, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-20T22:33:42.437 DEBUG:teuthology.orchestra.run.smithi037:> sudo pkill -f 'journalctl -f -n 0 -u ceph-f9a6357a-779c-11ef-baf5-efdc52797490@osd.3.service' 2024-09-20T22:33:42.476 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-20T22:33:42.477 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-09-20T22:33:42.477 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-09-20T22:33:42.477 DEBUG:teuthology.orchestra.run.smithi194:> sudo systemctl stop ceph-f9a6357a-779c-11ef-baf5-efdc52797490@osd.4 2024-09-20T22:33:42.483 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:33:42 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6[61138]: 2024-09-20T22:33:42.379+0000 7f595101c640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.37:6806 osd.0 since back 2024-09-20T22:33:14.174345+0000 front 2024-09-20T22:33:14.174182+0000 (oldest deadline 2024-09-20T22:33:38.274185+0000) 2024-09-20T22:33:42.483 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:33:42 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:33:42.177+0000 7f262c93f640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.37:6806 osd.0 since back 2024-09-20T22:33:15.226504+0000 front 2024-09-20T22:33:15.226384+0000 (oldest deadline 2024-09-20T22:33:39.326239+0000) 2024-09-20T22:33:42.781 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 20 22:33:42 smithi194 systemd[1]: Stopping Ceph osd.4 for f9a6357a-779c-11ef-baf5-efdc52797490... 2024-09-20T22:33:42.781 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:33:42 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-5[58907]: 2024-09-20T22:33:42.481+0000 7f63e8804640 -1 osd.5 162 heartbeat_check: no reply from 172.21.15.37:6806 osd.0 since back 2024-09-20T22:33:14.098648+0000 front 2024-09-20T22:33:14.098917+0000 (oldest deadline 2024-09-20T22:33:39.998833+0000) 2024-09-20T22:33:43.095 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 20 22:33:42 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-4[56643]: 2024-09-20T22:33:42.779+0000 7f8b0cf08640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-20T22:33:43.095 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 20 22:33:42 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-4[56643]: 2024-09-20T22:33:42.779+0000 7f8b0cf08640 -1 osd.4 162 *** Got signal Terminated *** 2024-09-20T22:33:43.095 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 20 22:33:42 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-4[56643]: 2024-09-20T22:33:42.779+0000 7f8b0cf08640 -1 osd.4 162 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-20T22:33:43.095 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 20 22:33:42 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-4[56643]: 2024-09-20T22:33:42.842+0000 7f8b05516640 -1 osd.4 162 heartbeat_check: no reply from 172.21.15.37:6806 osd.0 since back 2024-09-20T22:33:11.385940+0000 front 2024-09-20T22:33:11.385925+0000 (oldest deadline 2024-09-20T22:33:36.586139+0000) 2024-09-20T22:33:43.464 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:33:43 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6[61138]: 2024-09-20T22:33:43.413+0000 7f595101c640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.37:6806 osd.0 since back 2024-09-20T22:33:14.174345+0000 front 2024-09-20T22:33:14.174182+0000 (oldest deadline 2024-09-20T22:33:38.274185+0000) 2024-09-20T22:33:43.464 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:33:43 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:33:43.169+0000 7f262c93f640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.37:6806 osd.0 since back 2024-09-20T22:33:15.226504+0000 front 2024-09-20T22:33:15.226384+0000 (oldest deadline 2024-09-20T22:33:39.326239+0000) 2024-09-20T22:33:43.821 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:33:43 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-5[58907]: 2024-09-20T22:33:43.462+0000 7f63e8804640 -1 osd.5 162 heartbeat_check: no reply from 172.21.15.37:6806 osd.0 since back 2024-09-20T22:33:14.098648+0000 front 2024-09-20T22:33:14.098917+0000 (oldest deadline 2024-09-20T22:33:39.998833+0000) 2024-09-20T22:33:44.095 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 20 22:33:43 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-4[56643]: 2024-09-20T22:33:43.819+0000 7f8b05516640 -1 osd.4 162 heartbeat_check: no reply from 172.21.15.37:6806 osd.0 since back 2024-09-20T22:33:11.385940+0000 front 2024-09-20T22:33:11.385925+0000 (oldest deadline 2024-09-20T22:33:36.586139+0000) 2024-09-20T22:33:44.470 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:33:44 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6[61138]: 2024-09-20T22:33:44.421+0000 7f595101c640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.37:6806 osd.0 since back 2024-09-20T22:33:14.174345+0000 front 2024-09-20T22:33:14.174182+0000 (oldest deadline 2024-09-20T22:33:38.274185+0000) 2024-09-20T22:33:44.471 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:33:44 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:33:44.205+0000 7f262c93f640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.37:6806 osd.0 since back 2024-09-20T22:33:15.226504+0000 front 2024-09-20T22:33:15.226384+0000 (oldest deadline 2024-09-20T22:33:39.326239+0000) 2024-09-20T22:33:44.801 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:33:44 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-5[58907]: 2024-09-20T22:33:44.468+0000 7f63e8804640 -1 osd.5 162 heartbeat_check: no reply from 172.21.15.37:6806 osd.0 since back 2024-09-20T22:33:14.098648+0000 front 2024-09-20T22:33:14.098917+0000 (oldest deadline 2024-09-20T22:33:39.998833+0000) 2024-09-20T22:33:45.094 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 20 22:33:44 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-4[56643]: 2024-09-20T22:33:44.799+0000 7f8b05516640 -1 osd.4 162 heartbeat_check: no reply from 172.21.15.37:6806 osd.0 since back 2024-09-20T22:33:11.385940+0000 front 2024-09-20T22:33:11.385925+0000 (oldest deadline 2024-09-20T22:33:36.586139+0000) 2024-09-20T22:33:45.595 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:33:45 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-5[58907]: 2024-09-20T22:33:45.462+0000 7f63e8804640 -1 osd.5 162 heartbeat_check: no reply from 172.21.15.37:6806 osd.0 since back 2024-09-20T22:33:14.098648+0000 front 2024-09-20T22:33:14.098917+0000 (oldest deadline 2024-09-20T22:33:39.998833+0000) 2024-09-20T22:33:45.595 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:33:45 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:33:45.212+0000 7f262c93f640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.37:6806 osd.0 since back 2024-09-20T22:33:15.226504+0000 front 2024-09-20T22:33:15.226384+0000 (oldest deadline 2024-09-20T22:33:39.326239+0000) 2024-09-20T22:33:45.595 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:33:45 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6[61138]: 2024-09-20T22:33:45.388+0000 7f595101c640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.37:6806 osd.0 since back 2024-09-20T22:33:14.174345+0000 front 2024-09-20T22:33:14.174182+0000 (oldest deadline 2024-09-20T22:33:38.274185+0000) 2024-09-20T22:33:45.595 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:33:45 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6[61138]: 2024-09-20T22:33:45.388+0000 7f595101c640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.37:6814 osd.1 since back 2024-09-20T22:33:18.775113+0000 front 2024-09-20T22:33:18.775074+0000 (oldest deadline 2024-09-20T22:33:44.674978+0000) 2024-09-20T22:33:46.094 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 20 22:33:45 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-4[56643]: 2024-09-20T22:33:45.824+0000 7f8b05516640 -1 osd.4 162 heartbeat_check: no reply from 172.21.15.37:6806 osd.0 since back 2024-09-20T22:33:11.385940+0000 front 2024-09-20T22:33:11.385925+0000 (oldest deadline 2024-09-20T22:33:36.586139+0000) 2024-09-20T22:33:46.095 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 20 22:33:45 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-4[56643]: 2024-09-20T22:33:45.824+0000 7f8b05516640 -1 osd.4 162 heartbeat_check: no reply from 172.21.15.37:6814 osd.1 since back 2024-09-20T22:33:21.187579+0000 front 2024-09-20T22:33:21.187663+0000 (oldest deadline 2024-09-20T22:33:45.287498+0000) 2024-09-20T22:33:46.470 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:33:46 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:33:46.195+0000 7f262c93f640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.37:6806 osd.0 since back 2024-09-20T22:33:15.226504+0000 front 2024-09-20T22:33:15.226384+0000 (oldest deadline 2024-09-20T22:33:39.326239+0000) 2024-09-20T22:33:46.470 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:33:46 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6[61138]: 2024-09-20T22:33:46.394+0000 7f595101c640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.37:6806 osd.0 since back 2024-09-20T22:33:14.174345+0000 front 2024-09-20T22:33:14.174182+0000 (oldest deadline 2024-09-20T22:33:38.274185+0000) 2024-09-20T22:33:46.470 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:33:46 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6[61138]: 2024-09-20T22:33:46.394+0000 7f595101c640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.37:6814 osd.1 since back 2024-09-20T22:33:18.775113+0000 front 2024-09-20T22:33:18.775074+0000 (oldest deadline 2024-09-20T22:33:44.674978+0000) 2024-09-20T22:33:46.841 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:33:46 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-5[58907]: 2024-09-20T22:33:46.468+0000 7f63e8804640 -1 osd.5 162 heartbeat_check: no reply from 172.21.15.37:6806 osd.0 since back 2024-09-20T22:33:14.098648+0000 front 2024-09-20T22:33:14.098917+0000 (oldest deadline 2024-09-20T22:33:39.998833+0000) 2024-09-20T22:33:47.095 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 20 22:33:46 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-4[56643]: 2024-09-20T22:33:46.839+0000 7f8b05516640 -1 osd.4 162 heartbeat_check: no reply from 172.21.15.37:6806 osd.0 since back 2024-09-20T22:33:11.385940+0000 front 2024-09-20T22:33:11.385925+0000 (oldest deadline 2024-09-20T22:33:36.586139+0000) 2024-09-20T22:33:47.095 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 20 22:33:46 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-4[56643]: 2024-09-20T22:33:46.839+0000 7f8b05516640 -1 osd.4 162 heartbeat_check: no reply from 172.21.15.37:6814 osd.1 since back 2024-09-20T22:33:21.187579+0000 front 2024-09-20T22:33:21.187663+0000 (oldest deadline 2024-09-20T22:33:45.287498+0000) 2024-09-20T22:33:47.423 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:33:47 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:33:47.168+0000 7f262c93f640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.37:6806 osd.0 since back 2024-09-20T22:33:15.226504+0000 front 2024-09-20T22:33:15.226384+0000 (oldest deadline 2024-09-20T22:33:39.326239+0000) 2024-09-20T22:33:47.787 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:33:47 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6[61138]: 2024-09-20T22:33:47.421+0000 7f595101c640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.37:6806 osd.0 since back 2024-09-20T22:33:14.174345+0000 front 2024-09-20T22:33:14.174182+0000 (oldest deadline 2024-09-20T22:33:38.274185+0000) 2024-09-20T22:33:47.787 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:33:47 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6[61138]: 2024-09-20T22:33:47.421+0000 7f595101c640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.37:6814 osd.1 since back 2024-09-20T22:33:18.775113+0000 front 2024-09-20T22:33:18.775074+0000 (oldest deadline 2024-09-20T22:33:44.674978+0000) 2024-09-20T22:33:47.787 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:33:47 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-5[58907]: 2024-09-20T22:33:47.440+0000 7f63e8804640 -1 osd.5 162 heartbeat_check: no reply from 172.21.15.37:6806 osd.0 since back 2024-09-20T22:33:14.098648+0000 front 2024-09-20T22:33:14.098917+0000 (oldest deadline 2024-09-20T22:33:39.998833+0000) 2024-09-20T22:33:47.788 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:33:47 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-5[58907]: 2024-09-20T22:33:47.440+0000 7f63e8804640 -1 osd.5 162 heartbeat_check: no reply from 172.21.15.37:6814 osd.1 since back 2024-09-20T22:33:21.000609+0000 front 2024-09-20T22:33:21.000573+0000 (oldest deadline 2024-09-20T22:33:46.900484+0000) 2024-09-20T22:33:48.566 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 20 22:33:48 smithi194 podman[73802]: 2024-09-20 22:33:48.359576594 +0000 UTC m=+5.685335418 container died 641138d63e507c235a675f50a2e9c6dab458f29aa354837ee0ab3f9da58a9314 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-4, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, ceph=True, RELEASE=squid-9e2ac70, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-09-20T22:33:48.566 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:33:48 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-5[58907]: 2024-09-20T22:33:48.425+0000 7f63e8804640 -1 osd.5 162 heartbeat_check: no reply from 172.21.15.37:6806 osd.0 since back 2024-09-20T22:33:14.098648+0000 front 2024-09-20T22:33:14.098917+0000 (oldest deadline 2024-09-20T22:33:39.998833+0000) 2024-09-20T22:33:48.566 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:33:48 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-5[58907]: 2024-09-20T22:33:48.425+0000 7f63e8804640 -1 osd.5 162 heartbeat_check: no reply from 172.21.15.37:6814 osd.1 since back 2024-09-20T22:33:21.000609+0000 front 2024-09-20T22:33:21.000573+0000 (oldest deadline 2024-09-20T22:33:46.900484+0000) 2024-09-20T22:33:48.567 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:33:48 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6[61138]: 2024-09-20T22:33:48.434+0000 7f595101c640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.37:6806 osd.0 since back 2024-09-20T22:33:14.174345+0000 front 2024-09-20T22:33:14.174182+0000 (oldest deadline 2024-09-20T22:33:38.274185+0000) 2024-09-20T22:33:48.567 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:33:48 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6[61138]: 2024-09-20T22:33:48.434+0000 7f595101c640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.37:6814 osd.1 since back 2024-09-20T22:33:18.775113+0000 front 2024-09-20T22:33:18.775074+0000 (oldest deadline 2024-09-20T22:33:44.674978+0000) 2024-09-20T22:33:48.567 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:33:48 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:33:48.211+0000 7f262c93f640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.37:6806 osd.0 since back 2024-09-20T22:33:15.226504+0000 front 2024-09-20T22:33:15.226384+0000 (oldest deadline 2024-09-20T22:33:39.326239+0000) 2024-09-20T22:33:48.567 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:33:48 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:33:48.211+0000 7f262c93f640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.37:6814 osd.1 since back 2024-09-20T22:33:23.927833+0000 front 2024-09-20T22:33:23.927937+0000 (oldest deadline 2024-09-20T22:33:48.027693+0000) 2024-09-20T22:33:48.844 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 20 22:33:48 smithi194 podman[73802]: 2024-09-20 22:33:48.729861304 +0000 UTC m=+6.055620125 container cleanup 641138d63e507c235a675f50a2e9c6dab458f29aa354837ee0ab3f9da58a9314 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-4, GIT_CLEAN=True, RELEASE=squid-9e2ac70, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-09-20T22:33:48.845 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 20 22:33:48 smithi194 bash[73802]: ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-4 2024-09-20T22:33:49.343 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 20 22:33:48 smithi194 podman[73818]: 2024-09-20 22:33:48.981968988 +0000 UTC m=+0.622696031 container remove 641138d63e507c235a675f50a2e9c6dab458f29aa354837ee0ab3f9da58a9314 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-4, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, RELEASE=squid-9e2ac70) 2024-09-20T22:33:49.343 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:33:49 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:33:49.202+0000 7f262c93f640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.37:6806 osd.0 since back 2024-09-20T22:33:15.226504+0000 front 2024-09-20T22:33:15.226384+0000 (oldest deadline 2024-09-20T22:33:39.326239+0000) 2024-09-20T22:33:49.344 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:33:49 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:33:49.202+0000 7f262c93f640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.37:6814 osd.1 since back 2024-09-20T22:33:23.927833+0000 front 2024-09-20T22:33:23.927937+0000 (oldest deadline 2024-09-20T22:33:48.027693+0000) 2024-09-20T22:33:49.595 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 20 22:33:49 smithi194 podman[73906]: 2024-09-20 22:33:49.244967184 +0000 UTC m=+0.016097850 image pull cc9251b744a3faafca7c3095f79bfff81873afe823f31a03a11e015a34abcdbd quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5 2024-09-20T22:33:49.595 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 20 22:33:49 smithi194 podman[73906]: 2024-09-20 22:33:49.348921969 +0000 UTC m=+0.120052629 container create 48823cc708c0485d13508d57ac3e4b8fb712d1ce44ac1b9f98c987c0e5f1ab35 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-4-deactivate, GIT_CLEAN=True, RELEASE=squid-9e2ac70, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-09-20T22:33:49.595 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:33:49 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-5[58907]: 2024-09-20T22:33:49.433+0000 7f63e8804640 -1 osd.5 162 heartbeat_check: no reply from 172.21.15.37:6806 osd.0 since back 2024-09-20T22:33:14.098648+0000 front 2024-09-20T22:33:14.098917+0000 (oldest deadline 2024-09-20T22:33:39.998833+0000) 2024-09-20T22:33:49.595 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:33:49 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-5[58907]: 2024-09-20T22:33:49.433+0000 7f63e8804640 -1 osd.5 162 heartbeat_check: no reply from 172.21.15.37:6814 osd.1 since back 2024-09-20T22:33:21.000609+0000 front 2024-09-20T22:33:21.000573+0000 (oldest deadline 2024-09-20T22:33:46.900484+0000) 2024-09-20T22:33:49.596 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:33:49 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6[61138]: 2024-09-20T22:33:49.476+0000 7f595101c640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.37:6806 osd.0 since back 2024-09-20T22:33:14.174345+0000 front 2024-09-20T22:33:14.174182+0000 (oldest deadline 2024-09-20T22:33:38.274185+0000) 2024-09-20T22:33:49.596 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:33:49 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6[61138]: 2024-09-20T22:33:49.476+0000 7f595101c640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.37:6814 osd.1 since back 2024-09-20T22:33:18.775113+0000 front 2024-09-20T22:33:18.775074+0000 (oldest deadline 2024-09-20T22:33:44.674978+0000) 2024-09-20T22:33:50.095 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 20 22:33:49 smithi194 podman[73906]: 2024-09-20 22:33:49.630086306 +0000 UTC m=+0.401216970 container init 48823cc708c0485d13508d57ac3e4b8fb712d1ce44ac1b9f98c987c0e5f1ab35 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-4-deactivate, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, ceph=True, RELEASE=squid-9e2ac70, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-20T22:33:50.095 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 20 22:33:49 smithi194 podman[73906]: 2024-09-20 22:33:49.633465573 +0000 UTC m=+0.404596233 container start 48823cc708c0485d13508d57ac3e4b8fb712d1ce44ac1b9f98c987c0e5f1ab35 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-4-deactivate, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-9e2ac70) 2024-09-20T22:33:50.095 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 20 22:33:49 smithi194 podman[73906]: 2024-09-20 22:33:49.7098378 +0000 UTC m=+0.480968466 container attach 48823cc708c0485d13508d57ac3e4b8fb712d1ce44ac1b9f98c987c0e5f1ab35 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-4-deactivate, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=squid-9e2ac70, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2) 2024-09-20T22:33:50.095 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 20 22:33:49 smithi194 podman[73906]: 2024-09-20 22:33:49.827250186 +0000 UTC m=+0.598380846 container died 48823cc708c0485d13508d57ac3e4b8fb712d1ce44ac1b9f98c987c0e5f1ab35 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-4-deactivate, CEPH_POINT_RELEASE=, RELEASE=squid-9e2ac70, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-20T22:33:50.397 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:33:50 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:33:50.250+0000 7f262c93f640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.37:6806 osd.0 since back 2024-09-20T22:33:15.226504+0000 front 2024-09-20T22:33:15.226384+0000 (oldest deadline 2024-09-20T22:33:39.326239+0000) 2024-09-20T22:33:50.397 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:33:50 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:33:50.250+0000 7f262c93f640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.37:6814 osd.1 since back 2024-09-20T22:33:23.927833+0000 front 2024-09-20T22:33:23.927937+0000 (oldest deadline 2024-09-20T22:33:48.027693+0000) 2024-09-20T22:33:50.425 DEBUG:teuthology.orchestra.run.smithi194:> sudo pkill -f 'journalctl -f -n 0 -u ceph-f9a6357a-779c-11ef-baf5-efdc52797490@osd.4.service' 2024-09-20T22:33:50.506 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-20T22:33:50.506 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-09-20T22:33:50.506 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-09-20T22:33:50.506 DEBUG:teuthology.orchestra.run.smithi194:> sudo systemctl stop ceph-f9a6357a-779c-11ef-baf5-efdc52797490@osd.5 2024-09-20T22:33:50.673 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:33:50 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6[61138]: 2024-09-20T22:33:50.449+0000 7f595101c640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.37:6806 osd.0 since back 2024-09-20T22:33:14.174345+0000 front 2024-09-20T22:33:14.174182+0000 (oldest deadline 2024-09-20T22:33:38.274185+0000) 2024-09-20T22:33:50.674 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:33:50 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6[61138]: 2024-09-20T22:33:50.449+0000 7f595101c640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.37:6814 osd.1 since back 2024-09-20T22:33:18.775113+0000 front 2024-09-20T22:33:18.775074+0000 (oldest deadline 2024-09-20T22:33:44.674978+0000) 2024-09-20T22:33:50.674 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:33:50 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-5[58907]: 2024-09-20T22:33:50.482+0000 7f63e8804640 -1 osd.5 162 heartbeat_check: no reply from 172.21.15.37:6806 osd.0 since back 2024-09-20T22:33:14.098648+0000 front 2024-09-20T22:33:14.098917+0000 (oldest deadline 2024-09-20T22:33:39.998833+0000) 2024-09-20T22:33:50.674 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:33:50 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-5[58907]: 2024-09-20T22:33:50.482+0000 7f63e8804640 -1 osd.5 162 heartbeat_check: no reply from 172.21.15.37:6814 osd.1 since back 2024-09-20T22:33:21.000609+0000 front 2024-09-20T22:33:21.000573+0000 (oldest deadline 2024-09-20T22:33:46.900484+0000) 2024-09-20T22:33:50.674 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:33:50 smithi194 systemd[1]: Stopping Ceph osd.5 for f9a6357a-779c-11ef-baf5-efdc52797490... 2024-09-20T22:33:51.095 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:33:50 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-5[58907]: 2024-09-20T22:33:50.882+0000 7f63ec1ee640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-20T22:33:51.095 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:33:50 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-5[58907]: 2024-09-20T22:33:50.882+0000 7f63ec1ee640 -1 osd.5 162 *** Got signal Terminated *** 2024-09-20T22:33:51.095 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:33:50 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-5[58907]: 2024-09-20T22:33:50.882+0000 7f63ec1ee640 -1 osd.5 162 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-20T22:33:51.594 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:33:51 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:33:51.289+0000 7f262c93f640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.37:6806 osd.0 since back 2024-09-20T22:33:15.226504+0000 front 2024-09-20T22:33:15.226384+0000 (oldest deadline 2024-09-20T22:33:39.326239+0000) 2024-09-20T22:33:51.595 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:33:51 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:33:51.289+0000 7f262c93f640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.37:6814 osd.1 since back 2024-09-20T22:33:23.927833+0000 front 2024-09-20T22:33:23.927937+0000 (oldest deadline 2024-09-20T22:33:48.027693+0000) 2024-09-20T22:33:51.595 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:33:51 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-5[58907]: 2024-09-20T22:33:51.460+0000 7f63e8804640 -1 osd.5 162 heartbeat_check: no reply from 172.21.15.37:6806 osd.0 since back 2024-09-20T22:33:14.098648+0000 front 2024-09-20T22:33:14.098917+0000 (oldest deadline 2024-09-20T22:33:39.998833+0000) 2024-09-20T22:33:51.595 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:33:51 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-5[58907]: 2024-09-20T22:33:51.460+0000 7f63e8804640 -1 osd.5 162 heartbeat_check: no reply from 172.21.15.37:6814 osd.1 since back 2024-09-20T22:33:21.000609+0000 front 2024-09-20T22:33:21.000573+0000 (oldest deadline 2024-09-20T22:33:46.900484+0000) 2024-09-20T22:33:51.595 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:33:51 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6[61138]: 2024-09-20T22:33:51.462+0000 7f595101c640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.37:6806 osd.0 since back 2024-09-20T22:33:14.174345+0000 front 2024-09-20T22:33:14.174182+0000 (oldest deadline 2024-09-20T22:33:38.274185+0000) 2024-09-20T22:33:51.595 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:33:51 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6[61138]: 2024-09-20T22:33:51.462+0000 7f595101c640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.37:6814 osd.1 since back 2024-09-20T22:33:18.775113+0000 front 2024-09-20T22:33:18.775074+0000 (oldest deadline 2024-09-20T22:33:44.674978+0000) 2024-09-20T22:33:52.510 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:33:52 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6[61138]: 2024-09-20T22:33:52.487+0000 7f595101c640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.37:6806 osd.0 since back 2024-09-20T22:33:14.174345+0000 front 2024-09-20T22:33:14.174182+0000 (oldest deadline 2024-09-20T22:33:38.274185+0000) 2024-09-20T22:33:52.510 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:33:52 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6[61138]: 2024-09-20T22:33:52.487+0000 7f595101c640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.37:6814 osd.1 since back 2024-09-20T22:33:18.775113+0000 front 2024-09-20T22:33:18.775074+0000 (oldest deadline 2024-09-20T22:33:44.674978+0000) 2024-09-20T22:33:52.510 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:33:52 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6[61138]: 2024-09-20T22:33:52.487+0000 7f595101c640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.37:6822 osd.2 since back 2024-09-20T22:33:26.375936+0000 front 2024-09-20T22:33:26.376002+0000 (oldest deadline 2024-09-20T22:33:52.275863+0000) 2024-09-20T22:33:52.511 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:33:52 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:33:52.255+0000 7f262c93f640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.37:6806 osd.0 since back 2024-09-20T22:33:15.226504+0000 front 2024-09-20T22:33:15.226384+0000 (oldest deadline 2024-09-20T22:33:39.326239+0000) 2024-09-20T22:33:52.511 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:33:52 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:33:52.255+0000 7f262c93f640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.37:6814 osd.1 since back 2024-09-20T22:33:23.927833+0000 front 2024-09-20T22:33:23.927937+0000 (oldest deadline 2024-09-20T22:33:48.027693+0000) 2024-09-20T22:33:52.845 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:33:52 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-5[58907]: 2024-09-20T22:33:52.508+0000 7f63e8804640 -1 osd.5 162 heartbeat_check: no reply from 172.21.15.37:6806 osd.0 since back 2024-09-20T22:33:14.098648+0000 front 2024-09-20T22:33:14.098917+0000 (oldest deadline 2024-09-20T22:33:39.998833+0000) 2024-09-20T22:33:52.845 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:33:52 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-5[58907]: 2024-09-20T22:33:52.508+0000 7f63e8804640 -1 osd.5 162 heartbeat_check: no reply from 172.21.15.37:6814 osd.1 since back 2024-09-20T22:33:21.000609+0000 front 2024-09-20T22:33:21.000573+0000 (oldest deadline 2024-09-20T22:33:46.900484+0000) 2024-09-20T22:33:53.595 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:33:53 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-5[58907]: 2024-09-20T22:33:53.476+0000 7f63e8804640 -1 osd.5 162 heartbeat_check: no reply from 172.21.15.37:6806 osd.0 since back 2024-09-20T22:33:14.098648+0000 front 2024-09-20T22:33:14.098917+0000 (oldest deadline 2024-09-20T22:33:39.998833+0000) 2024-09-20T22:33:53.595 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:33:53 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-5[58907]: 2024-09-20T22:33:53.476+0000 7f63e8804640 -1 osd.5 162 heartbeat_check: no reply from 172.21.15.37:6814 osd.1 since back 2024-09-20T22:33:21.000609+0000 front 2024-09-20T22:33:21.000573+0000 (oldest deadline 2024-09-20T22:33:46.900484+0000) 2024-09-20T22:33:53.595 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:33:53 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:33:53.234+0000 7f262c93f640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.37:6806 osd.0 since back 2024-09-20T22:33:15.226504+0000 front 2024-09-20T22:33:15.226384+0000 (oldest deadline 2024-09-20T22:33:39.326239+0000) 2024-09-20T22:33:53.595 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:33:53 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:33:53.234+0000 7f262c93f640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.37:6814 osd.1 since back 2024-09-20T22:33:23.927833+0000 front 2024-09-20T22:33:23.927937+0000 (oldest deadline 2024-09-20T22:33:48.027693+0000) 2024-09-20T22:33:53.596 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:33:53 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6[61138]: 2024-09-20T22:33:53.463+0000 7f595101c640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.37:6806 osd.0 since back 2024-09-20T22:33:14.174345+0000 front 2024-09-20T22:33:14.174182+0000 (oldest deadline 2024-09-20T22:33:38.274185+0000) 2024-09-20T22:33:53.596 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:33:53 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6[61138]: 2024-09-20T22:33:53.463+0000 7f595101c640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.37:6814 osd.1 since back 2024-09-20T22:33:18.775113+0000 front 2024-09-20T22:33:18.775074+0000 (oldest deadline 2024-09-20T22:33:44.674978+0000) 2024-09-20T22:33:53.596 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:33:53 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6[61138]: 2024-09-20T22:33:53.463+0000 7f595101c640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.37:6822 osd.2 since back 2024-09-20T22:33:26.375936+0000 front 2024-09-20T22:33:26.376002+0000 (oldest deadline 2024-09-20T22:33:52.275863+0000) 2024-09-20T22:33:54.595 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:33:54 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-5[58907]: 2024-09-20T22:33:54.439+0000 7f63e8804640 -1 osd.5 162 heartbeat_check: no reply from 172.21.15.37:6806 osd.0 since back 2024-09-20T22:33:14.098648+0000 front 2024-09-20T22:33:14.098917+0000 (oldest deadline 2024-09-20T22:33:39.998833+0000) 2024-09-20T22:33:54.595 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:33:54 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-5[58907]: 2024-09-20T22:33:54.439+0000 7f63e8804640 -1 osd.5 162 heartbeat_check: no reply from 172.21.15.37:6814 osd.1 since back 2024-09-20T22:33:21.000609+0000 front 2024-09-20T22:33:21.000573+0000 (oldest deadline 2024-09-20T22:33:46.900484+0000) 2024-09-20T22:33:54.595 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:33:54 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:33:54.271+0000 7f262c93f640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.37:6806 osd.0 since back 2024-09-20T22:33:15.226504+0000 front 2024-09-20T22:33:15.226384+0000 (oldest deadline 2024-09-20T22:33:39.326239+0000) 2024-09-20T22:33:54.595 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:33:54 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:33:54.271+0000 7f262c93f640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.37:6814 osd.1 since back 2024-09-20T22:33:23.927833+0000 front 2024-09-20T22:33:23.927937+0000 (oldest deadline 2024-09-20T22:33:48.027693+0000) 2024-09-20T22:33:54.595 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:33:54 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6[61138]: 2024-09-20T22:33:54.484+0000 7f595101c640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.37:6806 osd.0 since back 2024-09-20T22:33:14.174345+0000 front 2024-09-20T22:33:14.174182+0000 (oldest deadline 2024-09-20T22:33:38.274185+0000) 2024-09-20T22:33:54.595 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:33:54 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6[61138]: 2024-09-20T22:33:54.484+0000 7f595101c640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.37:6814 osd.1 since back 2024-09-20T22:33:18.775113+0000 front 2024-09-20T22:33:18.775074+0000 (oldest deadline 2024-09-20T22:33:44.674978+0000) 2024-09-20T22:33:54.596 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:33:54 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6[61138]: 2024-09-20T22:33:54.484+0000 7f595101c640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.37:6822 osd.2 since back 2024-09-20T22:33:26.375936+0000 front 2024-09-20T22:33:26.376002+0000 (oldest deadline 2024-09-20T22:33:52.275863+0000) 2024-09-20T22:33:55.595 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:33:55 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:33:55.312+0000 7f262c93f640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.37:6806 osd.0 since back 2024-09-20T22:33:15.226504+0000 front 2024-09-20T22:33:15.226384+0000 (oldest deadline 2024-09-20T22:33:39.326239+0000) 2024-09-20T22:33:55.595 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:33:55 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:33:55.312+0000 7f262c93f640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.37:6814 osd.1 since back 2024-09-20T22:33:23.927833+0000 front 2024-09-20T22:33:23.927937+0000 (oldest deadline 2024-09-20T22:33:48.027693+0000) 2024-09-20T22:33:55.595 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:33:55 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-5[58907]: 2024-09-20T22:33:55.434+0000 7f63e8804640 -1 osd.5 162 heartbeat_check: no reply from 172.21.15.37:6806 osd.0 since back 2024-09-20T22:33:14.098648+0000 front 2024-09-20T22:33:14.098917+0000 (oldest deadline 2024-09-20T22:33:39.998833+0000) 2024-09-20T22:33:55.595 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:33:55 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-5[58907]: 2024-09-20T22:33:55.434+0000 7f63e8804640 -1 osd.5 162 heartbeat_check: no reply from 172.21.15.37:6814 osd.1 since back 2024-09-20T22:33:21.000609+0000 front 2024-09-20T22:33:21.000573+0000 (oldest deadline 2024-09-20T22:33:46.900484+0000) 2024-09-20T22:33:55.595 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:33:55 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-5[58907]: 2024-09-20T22:33:55.434+0000 7f63e8804640 -1 osd.5 162 heartbeat_check: no reply from 172.21.15.37:6822 osd.2 since back 2024-09-20T22:33:26.901027+0000 front 2024-09-20T22:33:26.901013+0000 (oldest deadline 2024-09-20T22:33:55.101264+0000) 2024-09-20T22:33:55.595 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:33:55 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6[61138]: 2024-09-20T22:33:55.531+0000 7f595101c640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.37:6806 osd.0 since back 2024-09-20T22:33:14.174345+0000 front 2024-09-20T22:33:14.174182+0000 (oldest deadline 2024-09-20T22:33:38.274185+0000) 2024-09-20T22:33:55.595 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:33:55 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6[61138]: 2024-09-20T22:33:55.531+0000 7f595101c640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.37:6814 osd.1 since back 2024-09-20T22:33:18.775113+0000 front 2024-09-20T22:33:18.775074+0000 (oldest deadline 2024-09-20T22:33:44.674978+0000) 2024-09-20T22:33:55.596 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:33:55 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6[61138]: 2024-09-20T22:33:55.531+0000 7f595101c640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.37:6822 osd.2 since back 2024-09-20T22:33:26.375936+0000 front 2024-09-20T22:33:26.376002+0000 (oldest deadline 2024-09-20T22:33:52.275863+0000) 2024-09-20T22:33:56.580 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:33:56 smithi194 podman[74010]: 2024-09-20 22:33:56.349292889 +0000 UTC m=+5.601408322 container died 9a2a44a8db216e2301d554f271b9c1da0661313f3af899aa5b9c3d932a1fb5b4 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-5, GIT_CLEAN=True, org.label-schema.build-date=20240731, RELEASE=squid-9e2ac70, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-20T22:33:56.580 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:33:56 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:33:56.279+0000 7f262c93f640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.37:6806 osd.0 since back 2024-09-20T22:33:15.226504+0000 front 2024-09-20T22:33:15.226384+0000 (oldest deadline 2024-09-20T22:33:39.326239+0000) 2024-09-20T22:33:56.580 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:33:56 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:33:56.279+0000 7f262c93f640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.37:6814 osd.1 since back 2024-09-20T22:33:23.927833+0000 front 2024-09-20T22:33:23.927937+0000 (oldest deadline 2024-09-20T22:33:48.027693+0000) 2024-09-20T22:33:56.580 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:33:56 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:33:56.279+0000 7f262c93f640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.37:6822 osd.2 since back 2024-09-20T22:33:28.028225+0000 front 2024-09-20T22:33:28.028282+0000 (oldest deadline 2024-09-20T22:33:56.228586+0000) 2024-09-20T22:33:56.844 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:33:56 smithi194 podman[74010]: 2024-09-20 22:33:56.716075991 +0000 UTC m=+5.968191460 container cleanup 9a2a44a8db216e2301d554f271b9c1da0661313f3af899aa5b9c3d932a1fb5b4 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-5, io.buildah.version=1.37.2, RELEASE=squid-9e2ac70, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-09-20T22:33:56.845 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:33:56 smithi194 bash[74010]: ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-5 2024-09-20T22:33:56.845 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:33:56 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6[61138]: 2024-09-20T22:33:56.578+0000 7f595101c640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.37:6806 osd.0 since back 2024-09-20T22:33:14.174345+0000 front 2024-09-20T22:33:14.174182+0000 (oldest deadline 2024-09-20T22:33:38.274185+0000) 2024-09-20T22:33:56.845 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:33:56 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6[61138]: 2024-09-20T22:33:56.578+0000 7f595101c640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.37:6814 osd.1 since back 2024-09-20T22:33:18.775113+0000 front 2024-09-20T22:33:18.775074+0000 (oldest deadline 2024-09-20T22:33:44.674978+0000) 2024-09-20T22:33:56.845 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:33:56 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6[61138]: 2024-09-20T22:33:56.578+0000 7f595101c640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.37:6822 osd.2 since back 2024-09-20T22:33:26.375936+0000 front 2024-09-20T22:33:26.376002+0000 (oldest deadline 2024-09-20T22:33:52.275863+0000) 2024-09-20T22:33:57.304 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:33:56 smithi194 podman[74070]: 2024-09-20 22:33:56.983939325 +0000 UTC m=+0.632205126 container remove 9a2a44a8db216e2301d554f271b9c1da0661313f3af899aa5b9c3d932a1fb5b4 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-5, RELEASE=squid-9e2ac70, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-09-20T22:33:57.594 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:33:57 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:33:57.304+0000 7f262c93f640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.37:6806 osd.0 since back 2024-09-20T22:33:15.226504+0000 front 2024-09-20T22:33:15.226384+0000 (oldest deadline 2024-09-20T22:33:39.326239+0000) 2024-09-20T22:33:57.595 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:33:57 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:33:57.304+0000 7f262c93f640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.37:6814 osd.1 since back 2024-09-20T22:33:23.927833+0000 front 2024-09-20T22:33:23.927937+0000 (oldest deadline 2024-09-20T22:33:48.027693+0000) 2024-09-20T22:33:57.595 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:33:57 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:33:57.304+0000 7f262c93f640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.37:6822 osd.2 since back 2024-09-20T22:33:28.028225+0000 front 2024-09-20T22:33:28.028282+0000 (oldest deadline 2024-09-20T22:33:56.228586+0000) 2024-09-20T22:33:57.595 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:33:57 smithi194 podman[74159]: 2024-09-20 22:33:57.206026737 +0000 UTC m=+0.015895921 image pull cc9251b744a3faafca7c3095f79bfff81873afe823f31a03a11e015a34abcdbd quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5 2024-09-20T22:33:57.595 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:33:57 smithi194 podman[74159]: 2024-09-20 22:33:57.326406579 +0000 UTC m=+0.136275761 container create aba335bd5dcc840196e6e9dbc781dbb159b8fda07d5036648f88f860e58c6b9e (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-5-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-9e2ac70, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=) 2024-09-20T22:33:57.595 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:33:57 smithi194 podman[74159]: 2024-09-20 22:33:57.545278819 +0000 UTC m=+0.355148011 container init aba335bd5dcc840196e6e9dbc781dbb159b8fda07d5036648f88f860e58c6b9e (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-5-deactivate, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, RELEASE=squid-9e2ac70) 2024-09-20T22:33:57.595 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:33:57 smithi194 podman[74159]: 2024-09-20 22:33:57.548479685 +0000 UTC m=+0.358348865 container start aba335bd5dcc840196e6e9dbc781dbb159b8fda07d5036648f88f860e58c6b9e (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-5-deactivate, ceph=True, RELEASE=squid-9e2ac70, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-20T22:33:58.025 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:33:57 smithi194 podman[74159]: 2024-09-20 22:33:57.637375788 +0000 UTC m=+0.447244976 container attach aba335bd5dcc840196e6e9dbc781dbb159b8fda07d5036648f88f860e58c6b9e (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-5-deactivate, ceph=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, RELEASE=squid-9e2ac70, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-09-20T22:33:58.025 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:33:57 smithi194 podman[74159]: 2024-09-20 22:33:57.733860821 +0000 UTC m=+0.543730008 container died aba335bd5dcc840196e6e9dbc781dbb159b8fda07d5036648f88f860e58c6b9e (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-5-deactivate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=squid-9e2ac70, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD) 2024-09-20T22:33:58.026 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:33:57 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6[61138]: 2024-09-20T22:33:57.617+0000 7f595101c640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.37:6806 osd.0 since back 2024-09-20T22:33:14.174345+0000 front 2024-09-20T22:33:14.174182+0000 (oldest deadline 2024-09-20T22:33:38.274185+0000) 2024-09-20T22:33:58.026 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:33:57 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6[61138]: 2024-09-20T22:33:57.617+0000 7f595101c640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.37:6814 osd.1 since back 2024-09-20T22:33:18.775113+0000 front 2024-09-20T22:33:18.775074+0000 (oldest deadline 2024-09-20T22:33:44.674978+0000) 2024-09-20T22:33:58.026 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:33:57 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6[61138]: 2024-09-20T22:33:57.617+0000 7f595101c640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.37:6822 osd.2 since back 2024-09-20T22:33:26.375936+0000 front 2024-09-20T22:33:26.376002+0000 (oldest deadline 2024-09-20T22:33:52.275863+0000) 2024-09-20T22:33:58.280 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 20 22:33:58 smithi194 podman[74183]: 2024-09-20 22:33:58.265801279 +0000 UTC m=+0.526090917 container remove aba335bd5dcc840196e6e9dbc781dbb159b8fda07d5036648f88f860e58c6b9e (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-5-deactivate, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, RELEASE=squid-9e2ac70) 2024-09-20T22:33:58.285 DEBUG:teuthology.orchestra.run.smithi194:> sudo pkill -f 'journalctl -f -n 0 -u ceph-f9a6357a-779c-11ef-baf5-efdc52797490@osd.5.service' 2024-09-20T22:33:58.324 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-20T22:33:58.324 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-09-20T22:33:58.325 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-09-20T22:33:58.325 DEBUG:teuthology.orchestra.run.smithi194:> sudo systemctl stop ceph-f9a6357a-779c-11ef-baf5-efdc52797490@osd.6 2024-09-20T22:33:58.595 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:33:58 smithi194 systemd[1]: Stopping Ceph osd.6 for f9a6357a-779c-11ef-baf5-efdc52797490... 2024-09-20T22:33:58.595 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:33:58 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:33:58.339+0000 7f262c93f640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.37:6806 osd.0 since back 2024-09-20T22:33:15.226504+0000 front 2024-09-20T22:33:15.226384+0000 (oldest deadline 2024-09-20T22:33:39.326239+0000) 2024-09-20T22:33:58.595 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:33:58 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:33:58.339+0000 7f262c93f640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.37:6814 osd.1 since back 2024-09-20T22:33:23.927833+0000 front 2024-09-20T22:33:23.927937+0000 (oldest deadline 2024-09-20T22:33:48.027693+0000) 2024-09-20T22:33:58.595 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:33:58 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:33:58.339+0000 7f262c93f640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.37:6822 osd.2 since back 2024-09-20T22:33:28.028225+0000 front 2024-09-20T22:33:28.028282+0000 (oldest deadline 2024-09-20T22:33:56.228586+0000) 2024-09-20T22:33:59.095 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:33:58 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6[61138]: 2024-09-20T22:33:58.644+0000 7f595101c640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.37:6806 osd.0 since back 2024-09-20T22:33:14.174345+0000 front 2024-09-20T22:33:14.174182+0000 (oldest deadline 2024-09-20T22:33:38.274185+0000) 2024-09-20T22:33:59.095 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:33:58 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6[61138]: 2024-09-20T22:33:58.644+0000 7f595101c640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.37:6814 osd.1 since back 2024-09-20T22:33:18.775113+0000 front 2024-09-20T22:33:18.775074+0000 (oldest deadline 2024-09-20T22:33:44.674978+0000) 2024-09-20T22:33:59.095 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:33:58 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6[61138]: 2024-09-20T22:33:58.644+0000 7f595101c640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.37:6822 osd.2 since back 2024-09-20T22:33:26.375936+0000 front 2024-09-20T22:33:26.376002+0000 (oldest deadline 2024-09-20T22:33:52.275863+0000) 2024-09-20T22:33:59.095 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:33:58 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6[61138]: 2024-09-20T22:33:58.677+0000 7f5954a06640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-20T22:33:59.095 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:33:58 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6[61138]: 2024-09-20T22:33:58.677+0000 7f5954a06640 -1 osd.6 162 *** Got signal Terminated *** 2024-09-20T22:33:59.095 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:33:58 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6[61138]: 2024-09-20T22:33:58.677+0000 7f5954a06640 -1 osd.6 162 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-20T22:33:59.595 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:33:59 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:33:59.313+0000 7f262c93f640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.37:6806 osd.0 since back 2024-09-20T22:33:15.226504+0000 front 2024-09-20T22:33:15.226384+0000 (oldest deadline 2024-09-20T22:33:39.326239+0000) 2024-09-20T22:33:59.595 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:33:59 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:33:59.313+0000 7f262c93f640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.37:6814 osd.1 since back 2024-09-20T22:33:23.927833+0000 front 2024-09-20T22:33:23.927937+0000 (oldest deadline 2024-09-20T22:33:48.027693+0000) 2024-09-20T22:33:59.595 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:33:59 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:33:59.313+0000 7f262c93f640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.37:6822 osd.2 since back 2024-09-20T22:33:28.028225+0000 front 2024-09-20T22:33:28.028282+0000 (oldest deadline 2024-09-20T22:33:56.228586+0000) 2024-09-20T22:34:00.095 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:33:59 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6[61138]: 2024-09-20T22:33:59.646+0000 7f595101c640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.37:6806 osd.0 since back 2024-09-20T22:33:14.174345+0000 front 2024-09-20T22:33:14.174182+0000 (oldest deadline 2024-09-20T22:33:38.274185+0000) 2024-09-20T22:34:00.095 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:33:59 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6[61138]: 2024-09-20T22:33:59.646+0000 7f595101c640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.37:6814 osd.1 since back 2024-09-20T22:33:18.775113+0000 front 2024-09-20T22:33:18.775074+0000 (oldest deadline 2024-09-20T22:33:44.674978+0000) 2024-09-20T22:34:00.095 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:33:59 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6[61138]: 2024-09-20T22:33:59.646+0000 7f595101c640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.37:6822 osd.2 since back 2024-09-20T22:33:26.375936+0000 front 2024-09-20T22:33:26.376002+0000 (oldest deadline 2024-09-20T22:33:52.275863+0000) 2024-09-20T22:34:00.594 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:34:00 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:34:00.283+0000 7f262c93f640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.37:6806 osd.0 since back 2024-09-20T22:33:15.226504+0000 front 2024-09-20T22:33:15.226384+0000 (oldest deadline 2024-09-20T22:33:39.326239+0000) 2024-09-20T22:34:00.595 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:34:00 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:34:00.283+0000 7f262c93f640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.37:6814 osd.1 since back 2024-09-20T22:33:23.927833+0000 front 2024-09-20T22:33:23.927937+0000 (oldest deadline 2024-09-20T22:33:48.027693+0000) 2024-09-20T22:34:00.595 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:34:00 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:34:00.283+0000 7f262c93f640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.37:6822 osd.2 since back 2024-09-20T22:33:28.028225+0000 front 2024-09-20T22:33:28.028282+0000 (oldest deadline 2024-09-20T22:33:56.228586+0000) 2024-09-20T22:34:01.094 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:34:00 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6[61138]: 2024-09-20T22:34:00.619+0000 7f595101c640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.37:6806 osd.0 since back 2024-09-20T22:33:14.174345+0000 front 2024-09-20T22:33:14.174182+0000 (oldest deadline 2024-09-20T22:33:38.274185+0000) 2024-09-20T22:34:01.095 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:34:00 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6[61138]: 2024-09-20T22:34:00.619+0000 7f595101c640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.37:6814 osd.1 since back 2024-09-20T22:33:18.775113+0000 front 2024-09-20T22:33:18.775074+0000 (oldest deadline 2024-09-20T22:33:44.674978+0000) 2024-09-20T22:34:01.095 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:34:00 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6[61138]: 2024-09-20T22:34:00.619+0000 7f595101c640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.37:6822 osd.2 since back 2024-09-20T22:33:26.375936+0000 front 2024-09-20T22:33:26.376002+0000 (oldest deadline 2024-09-20T22:33:52.275863+0000) 2024-09-20T22:34:01.594 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:34:01 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:34:01.286+0000 7f262c93f640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.37:6806 osd.0 since back 2024-09-20T22:33:15.226504+0000 front 2024-09-20T22:33:15.226384+0000 (oldest deadline 2024-09-20T22:33:39.326239+0000) 2024-09-20T22:34:01.595 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:34:01 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:34:01.286+0000 7f262c93f640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.37:6814 osd.1 since back 2024-09-20T22:33:23.927833+0000 front 2024-09-20T22:33:23.927937+0000 (oldest deadline 2024-09-20T22:33:48.027693+0000) 2024-09-20T22:34:01.595 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:34:01 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:34:01.286+0000 7f262c93f640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.37:6822 osd.2 since back 2024-09-20T22:33:28.028225+0000 front 2024-09-20T22:33:28.028282+0000 (oldest deadline 2024-09-20T22:33:56.228586+0000) 2024-09-20T22:34:01.595 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:34:01 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:34:01.286+0000 7f262c93f640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.37:6830 osd.3 since back 2024-09-20T22:33:36.229086+0000 front 2024-09-20T22:33:36.229105+0000 (oldest deadline 2024-09-20T22:34:00.328962+0000) 2024-09-20T22:34:02.094 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:34:01 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6[61138]: 2024-09-20T22:34:01.619+0000 7f595101c640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.37:6806 osd.0 since back 2024-09-20T22:33:14.174345+0000 front 2024-09-20T22:33:14.174182+0000 (oldest deadline 2024-09-20T22:33:38.274185+0000) 2024-09-20T22:34:02.095 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:34:01 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6[61138]: 2024-09-20T22:34:01.619+0000 7f595101c640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.37:6814 osd.1 since back 2024-09-20T22:33:18.775113+0000 front 2024-09-20T22:33:18.775074+0000 (oldest deadline 2024-09-20T22:33:44.674978+0000) 2024-09-20T22:34:02.095 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:34:01 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6[61138]: 2024-09-20T22:34:01.619+0000 7f595101c640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.37:6822 osd.2 since back 2024-09-20T22:33:26.375936+0000 front 2024-09-20T22:33:26.376002+0000 (oldest deadline 2024-09-20T22:33:52.275863+0000) 2024-09-20T22:34:02.594 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:34:02 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:34:02.267+0000 7f262c93f640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.37:6806 osd.0 since back 2024-09-20T22:33:15.226504+0000 front 2024-09-20T22:33:15.226384+0000 (oldest deadline 2024-09-20T22:33:39.326239+0000) 2024-09-20T22:34:02.595 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:34:02 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:34:02.267+0000 7f262c93f640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.37:6814 osd.1 since back 2024-09-20T22:33:23.927833+0000 front 2024-09-20T22:33:23.927937+0000 (oldest deadline 2024-09-20T22:33:48.027693+0000) 2024-09-20T22:34:02.595 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:34:02 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:34:02.267+0000 7f262c93f640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.37:6822 osd.2 since back 2024-09-20T22:33:28.028225+0000 front 2024-09-20T22:33:28.028282+0000 (oldest deadline 2024-09-20T22:33:56.228586+0000) 2024-09-20T22:34:02.595 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:34:02 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:34:02.267+0000 7f262c93f640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.37:6830 osd.3 since back 2024-09-20T22:33:36.229086+0000 front 2024-09-20T22:33:36.229105+0000 (oldest deadline 2024-09-20T22:34:00.328962+0000) 2024-09-20T22:34:03.094 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:34:02 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6[61138]: 2024-09-20T22:34:02.627+0000 7f595101c640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.37:6806 osd.0 since back 2024-09-20T22:33:14.174345+0000 front 2024-09-20T22:33:14.174182+0000 (oldest deadline 2024-09-20T22:33:38.274185+0000) 2024-09-20T22:34:03.095 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:34:02 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6[61138]: 2024-09-20T22:34:02.627+0000 7f595101c640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.37:6814 osd.1 since back 2024-09-20T22:33:18.775113+0000 front 2024-09-20T22:33:18.775074+0000 (oldest deadline 2024-09-20T22:33:44.674978+0000) 2024-09-20T22:34:03.095 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:34:02 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6[61138]: 2024-09-20T22:34:02.627+0000 7f595101c640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.37:6822 osd.2 since back 2024-09-20T22:33:26.375936+0000 front 2024-09-20T22:33:26.376002+0000 (oldest deadline 2024-09-20T22:33:52.275863+0000) 2024-09-20T22:34:03.595 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:34:03 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:34:03.253+0000 7f262c93f640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.37:6806 osd.0 since back 2024-09-20T22:33:15.226504+0000 front 2024-09-20T22:33:15.226384+0000 (oldest deadline 2024-09-20T22:33:39.326239+0000) 2024-09-20T22:34:03.595 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:34:03 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:34:03.253+0000 7f262c93f640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.37:6814 osd.1 since back 2024-09-20T22:33:23.927833+0000 front 2024-09-20T22:33:23.927937+0000 (oldest deadline 2024-09-20T22:33:48.027693+0000) 2024-09-20T22:34:03.595 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:34:03 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:34:03.253+0000 7f262c93f640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.37:6822 osd.2 since back 2024-09-20T22:33:28.028225+0000 front 2024-09-20T22:33:28.028282+0000 (oldest deadline 2024-09-20T22:33:56.228586+0000) 2024-09-20T22:34:03.595 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:34:03 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:34:03.253+0000 7f262c93f640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.37:6830 osd.3 since back 2024-09-20T22:33:36.229086+0000 front 2024-09-20T22:33:36.229105+0000 (oldest deadline 2024-09-20T22:34:00.328962+0000) 2024-09-20T22:34:04.095 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:34:03 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6[61138]: 2024-09-20T22:34:03.605+0000 7f595101c640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.37:6806 osd.0 since back 2024-09-20T22:33:14.174345+0000 front 2024-09-20T22:33:14.174182+0000 (oldest deadline 2024-09-20T22:33:38.274185+0000) 2024-09-20T22:34:04.095 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:34:03 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6[61138]: 2024-09-20T22:34:03.605+0000 7f595101c640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.37:6814 osd.1 since back 2024-09-20T22:33:18.775113+0000 front 2024-09-20T22:33:18.775074+0000 (oldest deadline 2024-09-20T22:33:44.674978+0000) 2024-09-20T22:34:04.095 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:34:03 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6[61138]: 2024-09-20T22:34:03.605+0000 7f595101c640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.37:6822 osd.2 since back 2024-09-20T22:33:26.375936+0000 front 2024-09-20T22:33:26.376002+0000 (oldest deadline 2024-09-20T22:33:52.275863+0000) 2024-09-20T22:34:04.438 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:34:04 smithi194 podman[74264]: 2024-09-20 22:34:04.166871005 +0000 UTC m=+5.612430567 container died 2daa9faae9a2ce6f545c58f6a5647a58ee50ef154591aaa2abfb315907bb4e40 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=squid-9e2ac70, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2) 2024-09-20T22:34:04.438 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:34:04 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:34:04.232+0000 7f262c93f640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.37:6806 osd.0 since back 2024-09-20T22:33:15.226504+0000 front 2024-09-20T22:33:15.226384+0000 (oldest deadline 2024-09-20T22:33:39.326239+0000) 2024-09-20T22:34:04.438 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:34:04 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:34:04.232+0000 7f262c93f640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.37:6814 osd.1 since back 2024-09-20T22:33:23.927833+0000 front 2024-09-20T22:33:23.927937+0000 (oldest deadline 2024-09-20T22:33:48.027693+0000) 2024-09-20T22:34:04.438 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:34:04 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:34:04.232+0000 7f262c93f640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.37:6822 osd.2 since back 2024-09-20T22:33:28.028225+0000 front 2024-09-20T22:33:28.028282+0000 (oldest deadline 2024-09-20T22:33:56.228586+0000) 2024-09-20T22:34:04.438 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:34:04 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:34:04.232+0000 7f262c93f640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.37:6830 osd.3 since back 2024-09-20T22:33:36.229086+0000 front 2024-09-20T22:33:36.229105+0000 (oldest deadline 2024-09-20T22:34:00.328962+0000) 2024-09-20T22:34:04.845 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:34:04 smithi194 podman[74264]: 2024-09-20 22:34:04.618540196 +0000 UTC m=+6.064099759 container cleanup 2daa9faae9a2ce6f545c58f6a5647a58ee50ef154591aaa2abfb315907bb4e40 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=squid-9e2ac70, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD) 2024-09-20T22:34:04.845 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:34:04 smithi194 bash[74264]: ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6 2024-09-20T22:34:05.230 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:34:04 smithi194 podman[74279]: 2024-09-20 22:34:04.894843636 +0000 UTC m=+0.720612927 container remove 2daa9faae9a2ce6f545c58f6a5647a58ee50ef154591aaa2abfb315907bb4e40 (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=squid-9e2ac70, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-09-20T22:34:05.482 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:34:05 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:34:05.254+0000 7f262c93f640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.37:6806 osd.0 since back 2024-09-20T22:33:15.226504+0000 front 2024-09-20T22:33:15.226384+0000 (oldest deadline 2024-09-20T22:33:39.326239+0000) 2024-09-20T22:34:05.482 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:34:05 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:34:05.254+0000 7f262c93f640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.37:6814 osd.1 since back 2024-09-20T22:33:23.927833+0000 front 2024-09-20T22:33:23.927937+0000 (oldest deadline 2024-09-20T22:33:48.027693+0000) 2024-09-20T22:34:05.482 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:34:05 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:34:05.254+0000 7f262c93f640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.37:6822 osd.2 since back 2024-09-20T22:33:28.028225+0000 front 2024-09-20T22:33:28.028282+0000 (oldest deadline 2024-09-20T22:33:56.228586+0000) 2024-09-20T22:34:05.483 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:34:05 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:34:05.254+0000 7f262c93f640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.37:6830 osd.3 since back 2024-09-20T22:33:36.229086+0000 front 2024-09-20T22:33:36.229105+0000 (oldest deadline 2024-09-20T22:34:00.328962+0000) 2024-09-20T22:34:05.483 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:34:05 smithi194 podman[74371]: 2024-09-20 22:34:05.229173902 +0000 UTC m=+0.112744663 container create d2b5bdc7e5fdedb95d424dfcd160afdd057f82a847fc78156867f65404f70e2b (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6-deactivate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=squid-9e2ac70, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS) 2024-09-20T22:34:05.483 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:34:05 smithi194 podman[74371]: 2024-09-20 22:34:05.132759118 +0000 UTC m=+0.016329876 image pull cc9251b744a3faafca7c3095f79bfff81873afe823f31a03a11e015a34abcdbd quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5 2024-09-20T22:34:05.483 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:34:05 smithi194 podman[74371]: 2024-09-20 22:34:05.481285554 +0000 UTC m=+0.364856317 container init d2b5bdc7e5fdedb95d424dfcd160afdd057f82a847fc78156867f65404f70e2b (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6-deactivate, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, RELEASE=squid-9e2ac70, GIT_CLEAN=True, ceph=True) 2024-09-20T22:34:05.844 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:34:05 smithi194 podman[74371]: 2024-09-20 22:34:05.484693478 +0000 UTC m=+0.368264240 container start d2b5bdc7e5fdedb95d424dfcd160afdd057f82a847fc78156867f65404f70e2b (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=squid-9e2ac70) 2024-09-20T22:34:05.845 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:34:05 smithi194 podman[74371]: 2024-09-20 22:34:05.564799787 +0000 UTC m=+0.448370549 container attach d2b5bdc7e5fdedb95d424dfcd160afdd057f82a847fc78156867f65404f70e2b (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6-deactivate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, RELEASE=squid-9e2ac70, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2) 2024-09-20T22:34:05.845 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:34:05 smithi194 podman[74371]: 2024-09-20 22:34:05.671365928 +0000 UTC m=+0.554936687 container died d2b5bdc7e5fdedb95d424dfcd160afdd057f82a847fc78156867f65404f70e2b (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6-deactivate, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=squid-9e2ac70, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, io.buildah.version=1.37.2) 2024-09-20T22:34:06.252 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 20 22:34:06 smithi194 podman[74393]: 2024-09-20 22:34:06.226244966 +0000 UTC m=+0.549202282 container remove d2b5bdc7e5fdedb95d424dfcd160afdd057f82a847fc78156867f65404f70e2b (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-6-deactivate, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=squid-9e2ac70, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-20T22:34:06.257 DEBUG:teuthology.orchestra.run.smithi194:> sudo pkill -f 'journalctl -f -n 0 -u ceph-f9a6357a-779c-11ef-baf5-efdc52797490@osd.6.service' 2024-09-20T22:34:06.297 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-20T22:34:06.297 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-09-20T22:34:06.297 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-09-20T22:34:06.298 DEBUG:teuthology.orchestra.run.smithi194:> sudo systemctl stop ceph-f9a6357a-779c-11ef-baf5-efdc52797490@osd.7 2024-09-20T22:34:06.594 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:34:06 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:34:06.272+0000 7f262c93f640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.37:6806 osd.0 since back 2024-09-20T22:33:15.226504+0000 front 2024-09-20T22:33:15.226384+0000 (oldest deadline 2024-09-20T22:33:39.326239+0000) 2024-09-20T22:34:06.595 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:34:06 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:34:06.272+0000 7f262c93f640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.37:6814 osd.1 since back 2024-09-20T22:33:23.927833+0000 front 2024-09-20T22:33:23.927937+0000 (oldest deadline 2024-09-20T22:33:48.027693+0000) 2024-09-20T22:34:06.595 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:34:06 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:34:06.272+0000 7f262c93f640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.37:6822 osd.2 since back 2024-09-20T22:33:28.028225+0000 front 2024-09-20T22:33:28.028282+0000 (oldest deadline 2024-09-20T22:33:56.228586+0000) 2024-09-20T22:34:06.595 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:34:06 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:34:06.272+0000 7f262c93f640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.37:6830 osd.3 since back 2024-09-20T22:33:36.229086+0000 front 2024-09-20T22:33:36.229105+0000 (oldest deadline 2024-09-20T22:34:00.328962+0000) 2024-09-20T22:34:06.595 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:34:06 smithi194 systemd[1]: Stopping Ceph osd.7 for f9a6357a-779c-11ef-baf5-efdc52797490... 2024-09-20T22:34:07.094 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:34:06 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:34:06.637+0000 7f2634331640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-20T22:34:07.095 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:34:06 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:34:06.637+0000 7f2634331640 -1 osd.7 162 *** Got signal Terminated *** 2024-09-20T22:34:07.095 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:34:06 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:34:06.637+0000 7f2634331640 -1 osd.7 162 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-20T22:34:07.595 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:34:07 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:34:07.262+0000 7f262c93f640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.37:6806 osd.0 since back 2024-09-20T22:33:15.226504+0000 front 2024-09-20T22:33:15.226384+0000 (oldest deadline 2024-09-20T22:33:39.326239+0000) 2024-09-20T22:34:07.595 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:34:07 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:34:07.262+0000 7f262c93f640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.37:6814 osd.1 since back 2024-09-20T22:33:23.927833+0000 front 2024-09-20T22:33:23.927937+0000 (oldest deadline 2024-09-20T22:33:48.027693+0000) 2024-09-20T22:34:07.595 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:34:07 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:34:07.262+0000 7f262c93f640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.37:6822 osd.2 since back 2024-09-20T22:33:28.028225+0000 front 2024-09-20T22:33:28.028282+0000 (oldest deadline 2024-09-20T22:33:56.228586+0000) 2024-09-20T22:34:07.595 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:34:07 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:34:07.262+0000 7f262c93f640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.37:6830 osd.3 since back 2024-09-20T22:33:36.229086+0000 front 2024-09-20T22:33:36.229105+0000 (oldest deadline 2024-09-20T22:34:00.328962+0000) 2024-09-20T22:34:08.548 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:34:08 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:34:08.220+0000 7f262c93f640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.37:6806 osd.0 since back 2024-09-20T22:33:15.226504+0000 front 2024-09-20T22:33:15.226384+0000 (oldest deadline 2024-09-20T22:33:39.326239+0000) 2024-09-20T22:34:08.549 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:34:08 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:34:08.220+0000 7f262c93f640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.37:6814 osd.1 since back 2024-09-20T22:33:23.927833+0000 front 2024-09-20T22:33:23.927937+0000 (oldest deadline 2024-09-20T22:33:48.027693+0000) 2024-09-20T22:34:08.549 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:34:08 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:34:08.220+0000 7f262c93f640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.37:6822 osd.2 since back 2024-09-20T22:33:28.028225+0000 front 2024-09-20T22:33:28.028282+0000 (oldest deadline 2024-09-20T22:33:56.228586+0000) 2024-09-20T22:34:08.549 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:34:08 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:34:08.220+0000 7f262c93f640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.37:6830 osd.3 since back 2024-09-20T22:33:36.229086+0000 front 2024-09-20T22:33:36.229105+0000 (oldest deadline 2024-09-20T22:34:00.328962+0000) 2024-09-20T22:34:09.595 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:34:09 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:34:09.256+0000 7f262c93f640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.37:6806 osd.0 since back 2024-09-20T22:33:15.226504+0000 front 2024-09-20T22:33:15.226384+0000 (oldest deadline 2024-09-20T22:33:39.326239+0000) 2024-09-20T22:34:09.595 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:34:09 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:34:09.256+0000 7f262c93f640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.37:6814 osd.1 since back 2024-09-20T22:33:23.927833+0000 front 2024-09-20T22:33:23.927937+0000 (oldest deadline 2024-09-20T22:33:48.027693+0000) 2024-09-20T22:34:09.595 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:34:09 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:34:09.256+0000 7f262c93f640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.37:6822 osd.2 since back 2024-09-20T22:33:28.028225+0000 front 2024-09-20T22:33:28.028282+0000 (oldest deadline 2024-09-20T22:33:56.228586+0000) 2024-09-20T22:34:09.595 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:34:09 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:34:09.256+0000 7f262c93f640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.37:6830 osd.3 since back 2024-09-20T22:33:36.229086+0000 front 2024-09-20T22:33:36.229105+0000 (oldest deadline 2024-09-20T22:34:00.328962+0000) 2024-09-20T22:34:10.595 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:34:10 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:34:10.253+0000 7f262c93f640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.37:6806 osd.0 since back 2024-09-20T22:33:15.226504+0000 front 2024-09-20T22:33:15.226384+0000 (oldest deadline 2024-09-20T22:33:39.326239+0000) 2024-09-20T22:34:10.595 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:34:10 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:34:10.253+0000 7f262c93f640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.37:6814 osd.1 since back 2024-09-20T22:33:23.927833+0000 front 2024-09-20T22:33:23.927937+0000 (oldest deadline 2024-09-20T22:33:48.027693+0000) 2024-09-20T22:34:10.595 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:34:10 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:34:10.253+0000 7f262c93f640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.37:6822 osd.2 since back 2024-09-20T22:33:28.028225+0000 front 2024-09-20T22:33:28.028282+0000 (oldest deadline 2024-09-20T22:33:56.228586+0000) 2024-09-20T22:34:10.595 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:34:10 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:34:10.253+0000 7f262c93f640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.37:6830 osd.3 since back 2024-09-20T22:33:36.229086+0000 front 2024-09-20T22:33:36.229105+0000 (oldest deadline 2024-09-20T22:34:00.328962+0000) 2024-09-20T22:34:11.594 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:34:11 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:34:11.259+0000 7f262c93f640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.37:6806 osd.0 since back 2024-09-20T22:33:15.226504+0000 front 2024-09-20T22:33:15.226384+0000 (oldest deadline 2024-09-20T22:33:39.326239+0000) 2024-09-20T22:34:11.595 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:34:11 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:34:11.259+0000 7f262c93f640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.37:6814 osd.1 since back 2024-09-20T22:33:23.927833+0000 front 2024-09-20T22:33:23.927937+0000 (oldest deadline 2024-09-20T22:33:48.027693+0000) 2024-09-20T22:34:11.595 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:34:11 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:34:11.259+0000 7f262c93f640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.37:6822 osd.2 since back 2024-09-20T22:33:28.028225+0000 front 2024-09-20T22:33:28.028282+0000 (oldest deadline 2024-09-20T22:33:56.228586+0000) 2024-09-20T22:34:11.595 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:34:11 smithi194 ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7[63404]: 2024-09-20T22:34:11.259+0000 7f262c93f640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.37:6830 osd.3 since back 2024-09-20T22:33:36.229086+0000 front 2024-09-20T22:33:36.229105+0000 (oldest deadline 2024-09-20T22:34:00.328962+0000) 2024-09-20T22:34:12.471 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:34:12 smithi194 podman[74471]: 2024-09-20 22:34:12.113617414 +0000 UTC m=+5.585035904 container died 281a2861e7e639e0d27e5914a35b818214cd1c2bd53d5dcd8efa0a7b39dc2eed (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, RELEASE=squid-9e2ac70, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-09-20T22:34:12.472 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:34:12 smithi194 podman[74471]: 2024-09-20 22:34:12.470738347 +0000 UTC m=+5.942156822 container cleanup 281a2861e7e639e0d27e5914a35b818214cd1c2bd53d5dcd8efa0a7b39dc2eed (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=squid-9e2ac70, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-09-20T22:34:12.731 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:34:12 smithi194 bash[74471]: ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7 2024-09-20T22:34:13.090 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:34:12 smithi194 podman[74498]: 2024-09-20 22:34:12.730390319 +0000 UTC m=+0.614598451 container remove 281a2861e7e639e0d27e5914a35b818214cd1c2bd53d5dcd8efa0a7b39dc2eed (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, RELEASE=squid-9e2ac70) 2024-09-20T22:34:13.345 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:34:13 smithi194 podman[74586]: 2024-09-20 22:34:12.992322626 +0000 UTC m=+0.016063739 image pull cc9251b744a3faafca7c3095f79bfff81873afe823f31a03a11e015a34abcdbd quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5 2024-09-20T22:34:13.345 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:34:13 smithi194 podman[74586]: 2024-09-20 22:34:13.09782117 +0000 UTC m=+0.121562320 container create 41fd9b3d01a9ee415b7c508aae8355dfe635fe74aed3882a341a6f1b596320ba (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7-deactivate, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-9e2ac70, org.label-schema.build-date=20240731, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-09-20T22:34:13.345 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:34:13 smithi194 podman[74586]: 2024-09-20 22:34:13.325371452 +0000 UTC m=+0.349112571 container init 41fd9b3d01a9ee415b7c508aae8355dfe635fe74aed3882a341a6f1b596320ba (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7-deactivate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, RELEASE=squid-9e2ac70) 2024-09-20T22:34:13.345 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:34:13 smithi194 podman[74586]: 2024-09-20 22:34:13.329010088 +0000 UTC m=+0.352751198 container start 41fd9b3d01a9ee415b7c508aae8355dfe635fe74aed3882a341a6f1b596320ba (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7-deactivate, RELEASE=squid-9e2ac70, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True, org.label-schema.schema-version=1.0) 2024-09-20T22:34:13.794 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:34:13 smithi194 podman[74586]: 2024-09-20 22:34:13.39165942 +0000 UTC m=+0.415400543 container attach 41fd9b3d01a9ee415b7c508aae8355dfe635fe74aed3882a341a6f1b596320ba (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, RELEASE=squid-9e2ac70, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-20T22:34:13.794 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 20 22:34:13 smithi194 podman[74586]: 2024-09-20 22:34:13.512064419 +0000 UTC m=+0.535805526 container died 41fd9b3d01a9ee415b7c508aae8355dfe635fe74aed3882a341a6f1b596320ba (image=quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5, name=ceph-f9a6357a-779c-11ef-baf5-efdc52797490-osd-7-deactivate, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, RELEASE=squid-9e2ac70, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-20T22:34:14.170 DEBUG:teuthology.orchestra.run.smithi194:> sudo pkill -f 'journalctl -f -n 0 -u ceph-f9a6357a-779c-11ef-baf5-efdc52797490@osd.7.service' 2024-09-20T22:34:14.216 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-20T22:34:14.216 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-09-20T22:34:14.216 DEBUG:teuthology.orchestra.run.smithi037:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid f9a6357a-779c-11ef-baf5-efdc52797490 --force --keep-logs 2024-09-20T22:37:15.989 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid f9a6357a-779c-11ef-baf5-efdc52797490 --force --keep-logs 2024-09-20T22:37:37.473 DEBUG:teuthology.orchestra.run.smithi037:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-20T22:37:37.508 DEBUG:teuthology.orchestra.run.smithi194:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-20T22:37:37.535 INFO:tasks.cephadm:Archiving crash dumps... 2024-09-20T22:37:37.537 DEBUG:teuthology.misc:Transferring archived files from smithi037:/var/lib/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/crash to /home/teuthworker/archive/teuthology-2024-09-20_21:08:02-orch-squid-distro-default-smithi/7912993/remote/smithi037/crash 2024-09-20T22:37:37.538 DEBUG:teuthology.orchestra.run.smithi037:> sudo tar c -f - -C /var/lib/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/crash -- . 2024-09-20T22:37:37.593 INFO:teuthology.orchestra.run.smithi037.stderr:tar: /var/lib/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/crash: Cannot open: No such file or directory 2024-09-20T22:37:37.593 INFO:teuthology.orchestra.run.smithi037.stderr:tar: Error is not recoverable: exiting now 2024-09-20T22:37:37.595 DEBUG:teuthology.misc:Transferring archived files from smithi194:/var/lib/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/crash to /home/teuthworker/archive/teuthology-2024-09-20_21:08:02-orch-squid-distro-default-smithi/7912993/remote/smithi194/crash 2024-09-20T22:37:37.596 DEBUG:teuthology.orchestra.run.smithi194:> sudo tar c -f - -C /var/lib/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/crash -- . 2024-09-20T22:37:37.665 INFO:teuthology.orchestra.run.smithi194.stderr:tar: /var/lib/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/crash: Cannot open: No such file or directory 2024-09-20T22:37:37.665 INFO:teuthology.orchestra.run.smithi194.stderr:tar: Error is not recoverable: exiting now 2024-09-20T22:37:37.667 INFO:tasks.cephadm:Checking cluster log for badness... 2024-09-20T22:37:37.667 DEBUG:teuthology.orchestra.run.smithi037:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/ceph.log | egrep CEPHADM_ | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v CEPHADM_REFRESH_FAILED | head -n 1 2024-09-20T22:37:37.698 INFO:tasks.cephadm:Compressing logs... 2024-09-20T22:37:37.698 DEBUG:teuthology.orchestra.run.smithi037:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-20T22:37:37.741 DEBUG:teuthology.orchestra.run.smithi194:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-20T22:37:37.765 INFO:teuthology.orchestra.run.smithi194.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/ceph-mgr.smithi194.bkcglt.log 2024-09-20T22:37:37.765 INFO:teuthology.orchestra.run.smithi194.stderr:: No such file or directory 2024-09-20T22:37:37.769 INFO:teuthology.orchestra.run.smithi037.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/ceph-osd.0.log 2024-09-20T22:37:37.769 INFO:teuthology.orchestra.run.smithi037.stderr:: No such file or directory 2024-09-20T22:37:37.769 INFO:teuthology.orchestra.run.smithi037.stderr:gzip -5 --verbose -- /var/log/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/ceph-mds.foofs.smithi037.sdtedi.log 2024-09-20T22:37:37.769 INFO:teuthology.orchestra.run.smithi037.stderr:gzip -5 --verbose -- /var/log/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/ceph-osd.1.log 2024-09-20T22:37:37.769 INFO:teuthology.orchestra.run.smithi037.stderr:/var/log/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/ceph-osd.0.log: /var/log/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/ceph-mds.foofs.smithi037.sdtedi.log: gzip -5 --verbose -- /var/log/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/ceph-mon.smithi037.log 2024-09-20T22:37:37.769 INFO:teuthology.orchestra.run.smithi037.stderr:/var/log/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/ceph-volume.log 2024-09-20T22:37:37.769 INFO:teuthology.orchestra.run.smithi037.stderr:gzip -5 --verbose -- /var/log/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/ceph.audit.log 2024-09-20T22:37:37.769 INFO:teuthology.orchestra.run.smithi037.stderr:/var/log/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/ceph-mon.smithi037.log: /var/log/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/ceph-mgr.smithi037.lykblh.log 2024-09-20T22:37:37.769 INFO:teuthology.orchestra.run.smithi037.stderr:/var/log/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/ceph.cephadm.log 2024-09-20T22:37:37.770 INFO:teuthology.orchestra.run.smithi037.stderr:/var/log/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/ceph-mgr.smithi037.lykblh.log: gzip -5 --verbose -- /var/log/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/ceph-osd.2.log 2024-09-20T22:37:37.771 INFO:teuthology.orchestra.run.smithi194.stderr:gzip -5 --verbose -- /var/log/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/ceph-mds.foofs.smithi194.lkroso.log 2024-09-20T22:37:37.771 INFO:teuthology.orchestra.run.smithi194.stderr:/var/log/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/ceph-mgr.smithi194.bkcglt.log: gzip -5 --verbose -- /var/log/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/ceph-osd.7.log 2024-09-20T22:37:37.771 INFO:teuthology.orchestra.run.smithi194.stderr:/var/log/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/ceph-mds.foofs.smithi194.lkroso.log: gzip -5 --verbose -- /var/log/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/ceph-volume.log 2024-09-20T22:37:37.772 INFO:teuthology.orchestra.run.smithi194.stderr:/var/log/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/ceph-osd.7.log: 87.6% -- replaced with /var/log/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/ceph-mds.foofs.smithi194.lkroso.log.gz 2024-09-20T22:37:37.772 INFO:teuthology.orchestra.run.smithi194.stderr:gzip -5 --verbose -- /var/log/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/ceph-mon.smithi194.log 2024-09-20T22:37:37.772 INFO:teuthology.orchestra.run.smithi194.stderr:/var/log/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/ceph-osd.4.log 2024-09-20T22:37:37.772 INFO:teuthology.orchestra.run.smithi194.stderr:/var/log/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/ceph-mon.smithi194.log: gzip -5 --verbose -- /var/log/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/ceph.audit.log 2024-09-20T22:37:37.772 INFO:teuthology.orchestra.run.smithi194.stderr:/var/log/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/ceph-osd.4.log: gzip -5 --verbose -- /var/log/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/ceph.cephadm.log 2024-09-20T22:37:37.772 INFO:teuthology.orchestra.run.smithi194.stderr:/var/log/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/ceph-osd.6.log 2024-09-20T22:37:37.772 INFO:teuthology.orchestra.run.smithi194.stderr:/var/log/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/ceph-osd.5.log 2024-09-20T22:37:37.772 INFO:teuthology.orchestra.run.smithi194.stderr: 85.6%/var/log/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/ceph-osd.6.log: -- replaced with /var/log/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/ceph.cephadm.log.gz 2024-09-20T22:37:37.772 INFO:teuthology.orchestra.run.smithi194.stderr:gzip -5 --verbose -- /var/log/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/ceph.log 2024-09-20T22:37:37.773 INFO:teuthology.orchestra.run.smithi194.stderr:/var/log/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/ceph-osd.5.log: 91.6% -- replaced with /var/log/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/ceph.audit.log.gz 2024-09-20T22:37:37.773 INFO:teuthology.orchestra.run.smithi037.stderr:/var/log/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/ceph.cephadm.log: 85.7% -- replaced with /var/log/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/ceph.cephadm.log.gz 2024-09-20T22:37:37.774 INFO:teuthology.orchestra.run.smithi037.stderr:gzip -5 --verbose -- /var/log/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/ceph.log 2024-09-20T22:37:37.774 INFO:teuthology.orchestra.run.smithi037.stderr:/var/log/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/ceph-osd.2.log: 91.5% -- replaced with /var/log/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/ceph.audit.log.gz 2024-09-20T22:37:37.777 INFO:teuthology.orchestra.run.smithi194.stderr: 91.2% -- replaced with /var/log/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/ceph-volume.log.gz 2024-09-20T22:37:37.778 INFO:teuthology.orchestra.run.smithi037.stderr:gzip -5 --verbose -- /var/log/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/ceph-osd.3.log 2024-09-20T22:37:37.779 INFO:teuthology.orchestra.run.smithi037.stderr:/var/log/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/ceph.log: 91.1% -- replaced with /var/log/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/ceph-volume.log.gz 2024-09-20T22:37:37.782 INFO:teuthology.orchestra.run.smithi037.stderr: 87.0% -- replaced with /var/log/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/ceph.log.gz 2024-09-20T22:37:37.788 INFO:teuthology.orchestra.run.smithi194.stderr:gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-09-20T22:37:37.792 INFO:teuthology.orchestra.run.smithi194.stderr:/var/log/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/ceph.log: /var/log/ceph/cephadm.log: 86.9% -- replaced with /var/log/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/ceph.log.gz 2024-09-20T22:37:37.792 INFO:teuthology.orchestra.run.smithi037.stderr: 91.4% -- replaced with /var/log/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/ceph-mds.foofs.smithi037.sdtedi.log.gz 2024-09-20T22:37:37.792 INFO:teuthology.orchestra.run.smithi037.stderr:gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-09-20T22:37:37.794 INFO:teuthology.orchestra.run.smithi194.stderr: 90.2% -- replaced with /var/log/ceph/cephadm.log.gz 2024-09-20T22:37:37.831 INFO:teuthology.orchestra.run.smithi037.stderr:/var/log/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/ceph-osd.3.log: /var/log/ceph/cephadm.log: 91.9% -- replaced with /var/log/ceph/cephadm.log.gz 2024-09-20T22:37:37.834 INFO:teuthology.orchestra.run.smithi194.stderr: 89.9% -- replaced with /var/log/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/ceph-mgr.smithi194.bkcglt.log.gz 2024-09-20T22:37:38.114 INFO:teuthology.orchestra.run.smithi194.stderr: 92.6% -- replaced with /var/log/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/ceph-mon.smithi194.log.gz 2024-09-20T22:37:38.159 INFO:teuthology.orchestra.run.smithi037.stderr: 88.6% -- replaced with /var/log/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/ceph-mgr.smithi037.lykblh.log.gz 2024-09-20T22:37:39.083 INFO:teuthology.orchestra.run.smithi037.stderr: 91.9% -- replaced with /var/log/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/ceph-mon.smithi037.log.gz 2024-09-20T22:37:50.588 INFO:teuthology.orchestra.run.smithi037.stderr: 94.4% -- replaced with /var/log/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/ceph-osd.2.log.gz 2024-09-20T22:37:53.010 INFO:teuthology.orchestra.run.smithi194.stderr: 93.9% -- replaced with /var/log/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/ceph-osd.4.log.gz 2024-09-20T22:37:53.300 INFO:teuthology.orchestra.run.smithi194.stderr: 94.3% -- replaced with /var/log/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/ceph-osd.6.log.gz 2024-09-20T22:37:54.537 INFO:teuthology.orchestra.run.smithi037.stderr: 94.1% -- replaced with /var/log/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/ceph-osd.0.log.gz 2024-09-20T22:37:56.364 INFO:teuthology.orchestra.run.smithi037.stderr: 94.1% -- replaced with /var/log/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/ceph-osd.3.log.gz 2024-09-20T22:37:57.037 INFO:teuthology.orchestra.run.smithi194.stderr: 94.2% -- replaced with /var/log/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/ceph-osd.5.log.gz 2024-09-20T22:37:57.335 INFO:teuthology.orchestra.run.smithi037.stderr: 94.2% -- replaced with /var/log/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/ceph-osd.1.log.gz 2024-09-20T22:37:57.337 INFO:teuthology.orchestra.run.smithi037.stderr: 2024-09-20T22:37:57.337 INFO:teuthology.orchestra.run.smithi037.stderr:real 0m19.582s 2024-09-20T22:37:57.337 INFO:teuthology.orchestra.run.smithi037.stderr:user 1m6.995s 2024-09-20T22:37:57.337 INFO:teuthology.orchestra.run.smithi037.stderr:sys 0m2.418s 2024-09-20T22:38:00.547 INFO:teuthology.orchestra.run.smithi194.stderr: 94.1% -- replaced with /var/log/ceph/f9a6357a-779c-11ef-baf5-efdc52797490/ceph-osd.7.log.gz 2024-09-20T22:38:00.549 INFO:teuthology.orchestra.run.smithi194.stderr: 2024-09-20T22:38:00.549 INFO:teuthology.orchestra.run.smithi194.stderr:real 0m22.794s 2024-09-20T22:38:00.549 INFO:teuthology.orchestra.run.smithi194.stderr:user 1m10.285s 2024-09-20T22:38:00.549 INFO:teuthology.orchestra.run.smithi194.stderr:sys 0m2.796s 2024-09-20T22:38:00.550 INFO:tasks.cephadm:Archiving logs... 2024-09-20T22:38:00.550 DEBUG:teuthology.misc:Transferring archived files from smithi037:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-09-20_21:08:02-orch-squid-distro-default-smithi/7912993/remote/smithi037/log 2024-09-20T22:38:00.551 DEBUG:teuthology.orchestra.run.smithi037:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-20T22:38:09.035 DEBUG:teuthology.misc:Transferring archived files from smithi194:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-09-20_21:08:02-orch-squid-distro-default-smithi/7912993/remote/smithi194/log 2024-09-20T22:38:09.036 DEBUG:teuthology.orchestra.run.smithi194:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-20T22:38:18.479 INFO:tasks.cephadm:Removing cluster... 2024-09-20T22:38:18.479 DEBUG:teuthology.orchestra.run.smithi037:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid f9a6357a-779c-11ef-baf5-efdc52797490 --force 2024-09-20T22:38:18.856 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid f9a6357a-779c-11ef-baf5-efdc52797490 --force 2024-09-20T22:38:19.221 INFO:tasks.cephadm:Removing cephadm ... 2024-09-20T22:38:19.221 DEBUG:teuthology.orchestra.run.smithi037:> rm -rf /home/ubuntu/cephtest/cephadm 2024-09-20T22:38:19.238 DEBUG:teuthology.orchestra.run.smithi194:> rm -rf /home/ubuntu/cephtest/cephadm 2024-09-20T22:38:19.252 INFO:tasks.cephadm:Teardown complete 2024-09-20T22:38:19.252 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-09-20T22:38:19.262 INFO:teuthology.task.clock:Checking final clock skew... 2024-09-20T22:38:19.263 DEBUG:teuthology.orchestra.run.smithi037:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-20T22:38:19.281 DEBUG:teuthology.orchestra.run.smithi194:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-20T22:38:19.294 INFO:teuthology.orchestra.run.smithi037.stderr:bash: line 1: ntpq: command not found 2024-09-20T22:38:19.299 INFO:teuthology.orchestra.run.smithi037.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-20T22:38:19.299 INFO:teuthology.orchestra.run.smithi037.stdout:=============================================================================== 2024-09-20T22:38:19.299 INFO:teuthology.orchestra.run.smithi037.stdout:^* hv01.front.sepia.ceph.com 3 6 377 58 -330us[ -285us] +/- 40ms 2024-09-20T22:38:19.299 INFO:teuthology.orchestra.run.smithi037.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 54 -56us[ -56us] +/- 61ms 2024-09-20T22:38:19.299 INFO:teuthology.orchestra.run.smithi037.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 56 +733us[ +733us] +/- 65ms 2024-09-20T22:38:19.299 INFO:teuthology.orchestra.run.smithi037.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-09-20T22:38:19.306 INFO:teuthology.orchestra.run.smithi194.stderr:bash: line 1: ntpq: command not found 2024-09-20T22:38:19.310 INFO:teuthology.orchestra.run.smithi194.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-20T22:38:19.310 INFO:teuthology.orchestra.run.smithi194.stdout:=============================================================================== 2024-09-20T22:38:19.310 INFO:teuthology.orchestra.run.smithi194.stdout:^* hv01.front.sepia.ceph.com 3 6 377 57 -340us[ -300us] +/- 40ms 2024-09-20T22:38:19.310 INFO:teuthology.orchestra.run.smithi194.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 58 -77us[ -38us] +/- 61ms 2024-09-20T22:38:19.310 INFO:teuthology.orchestra.run.smithi194.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 53 +710us[ +710us] +/- 65ms 2024-09-20T22:38:19.310 INFO:teuthology.orchestra.run.smithi194.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-09-20T22:38:19.314 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-09-20T22:38:19.324 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-09-20T22:38:19.324 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-09-20T22:38:19.334 DEBUG:teuthology.orchestra.run.smithi037:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-09-20T22:38:19.424 DEBUG:teuthology.orchestra.run.smithi194:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-09-20T22:38:19.497 DEBUG:teuthology.orchestra.run.smithi037:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-09-20T22:38:19.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-20T22:38:19.528 DEBUG:teuthology.orchestra.run.smithi194:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-09-20T22:38:19.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-20T22:38:19.557 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-09-20T22:38:19.567 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-09-20T22:38:19.576 INFO:teuthology.task.internal:Duration was 1971.261513 seconds 2024-09-20T22:38:19.576 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-09-20T22:38:19.594 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-09-20T22:38:19.594 DEBUG:teuthology.orchestra.run.smithi037:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-20T22:38:19.596 DEBUG:teuthology.orchestra.run.smithi194:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-20T22:38:19.635 INFO:teuthology.orchestra.run.smithi037.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-20T22:38:19.636 INFO:teuthology.orchestra.run.smithi194.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-20T22:38:19.953 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-09-20T22:38:19.953 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi037.front.sepia.ceph.com 2024-09-20T22:38:19.953 DEBUG:teuthology.orchestra.run.smithi037:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-20T22:38:20.016 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi194.front.sepia.ceph.com 2024-09-20T22:38:20.016 DEBUG:teuthology.orchestra.run.smithi194:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-20T22:38:20.044 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-09-20T22:38:20.044 DEBUG:teuthology.orchestra.run.smithi037:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-20T22:38:20.059 DEBUG:teuthology.orchestra.run.smithi194:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-20T22:38:20.111 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-09-20T22:38:20.111 DEBUG:teuthology.orchestra.run.smithi037:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-20T22:38:20.127 DEBUG:teuthology.orchestra.run.smithi194:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-20T22:38:34.373 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-09-20T22:38:34.390 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-09-20T22:38:34.390 DEBUG:teuthology.orchestra.run.smithi037:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-20T22:38:34.416 DEBUG:teuthology.orchestra.run.smithi194:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-20T22:38:34.446 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-09-20T22:38:34.457 DEBUG:teuthology.orchestra.run.smithi037:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-20T22:38:34.459 DEBUG:teuthology.orchestra.run.smithi194:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-20T22:38:34.481 INFO:teuthology.orchestra.run.smithi037.stdout:kernel.core_pattern = core 2024-09-20T22:38:34.513 INFO:teuthology.orchestra.run.smithi194.stdout:kernel.core_pattern = core 2024-09-20T22:38:34.547 DEBUG:teuthology.orchestra.run.smithi037:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-20T22:38:34.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-20T22:38:34.577 DEBUG:teuthology.orchestra.run.smithi194:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-20T22:38:34.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-20T22:38:34.602 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-09-20T22:38:34.613 INFO:teuthology.task.internal:Transferring archived files... 2024-09-20T22:38:34.613 DEBUG:teuthology.misc:Transferring archived files from smithi037:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-09-20_21:08:02-orch-squid-distro-default-smithi/7912993/remote/smithi037 2024-09-20T22:38:34.613 DEBUG:teuthology.orchestra.run.smithi037:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-20T22:38:34.676 DEBUG:teuthology.misc:Transferring archived files from smithi194:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-09-20_21:08:02-orch-squid-distro-default-smithi/7912993/remote/smithi194 2024-09-20T22:38:34.676 DEBUG:teuthology.orchestra.run.smithi194:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-20T22:38:34.719 INFO:teuthology.task.internal:Removing archive directory... 2024-09-20T22:38:34.719 DEBUG:teuthology.orchestra.run.smithi037:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-20T22:38:34.721 DEBUG:teuthology.orchestra.run.smithi194:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-20T22:38:34.769 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-09-20T22:38:34.780 INFO:teuthology.task.internal:Not uploading archives. 2024-09-20T22:38:34.781 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-09-20T22:38:34.852 INFO:teuthology.task.internal:Tidying up after the test... 2024-09-20T22:38:34.853 DEBUG:teuthology.orchestra.run.smithi037:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-20T22:38:34.855 DEBUG:teuthology.orchestra.run.smithi194:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-20T22:38:34.868 INFO:teuthology.orchestra.run.smithi037.stdout: 265324 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 20 22:38 /home/ubuntu/cephtest 2024-09-20T22:38:34.870 INFO:teuthology.orchestra.run.smithi194.stdout: 265324 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 20 22:38 /home/ubuntu/cephtest 2024-09-20T22:38:34.871 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-09-20T22:38:34.882 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-09-20T22:38:34.906 INFO:teuthology.run:Summary data: description: orch/cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} duration: 1971.2615129947662 owner: scheduled_teuthology@teuthology success: true 2024-09-20T22:38:34.906 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-20T22:38:34.975 INFO:teuthology.run:pass